zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: Support secure binary signing on protected pipelines (#30753)

Browse Source 
This PR supports the creation of securely signed binaries built from spack
develop as well as release branches and tags. Specifically:

- remove internal pr mirror url generation logic in favor of buildcache destination
on command line
    - with a single mirror url specified in the spack.yaml, this makes it clearer where 
    binaries from various pipelines are pushed
- designate some tags as reserved: ['public', 'protected', 'notary']
    - these tags are stripped from all jobs by default and provisioned internally
    based on pipeline type
- update gitlab ci yaml to include pipelines on more protected branches than just
develop (so include releases and tags)
    - binaries from all protected pipelines are pushed into mirrors including the
    branch name so releases, tags, and develop binaries are kept separate
- update rebuild jobs running on protected pipelines to run on special runners
provisioned with an intermediate signing key
    - protected rebuild jobs no longer use "SPACK_SIGNING_KEY" env var to
    obtain signing key (in fact, final signing key is nowhere available to rebuild jobs)
    - these intermediate signatures are verified at the end of each pipeline by a new
    signing job to ensure binaries were produced by a protected pipeline
- optionallly schedule a signing/notary job at the end of the pipeline to sign all
packges in the mirror
    - add signing-job-attributes to gitlab-ci section of spack environment to allow
    configuration
    - signing job runs on special runner (separate from protected rebuild runners)
    provisioned with public intermediate key and secret signing key
This commit is contained in:
Scott Wittenburg
2022-05-26 08:31:22 -06:00
committed by GitHub
parent b5a519fa51
commit 85e13260cf
13 changed files with 534 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
View File

@@ -29,7 +29,7 @@ spack:
- - $default_specs
- - $arch
mirrors: { "mirror": "s3://spack-binaries/build_systems" }
mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
gitlab-ci:
script:
@@ -38,6 +38,8 @@ spack:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- spack -d ci rebuild
image:
@@ -48,7 +50,7 @@ spack:
- match:
- cmake
runner-attributes:
tags: [ "spack", "public", "large", "x86_64"]
tags: [ "spack", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
@@ -61,7 +63,7 @@ spack:
- openjpeg
- sqlite
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
tags: [ "spack", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
@@ -85,7 +87,7 @@ spack:
- xz
- zlib
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
tags: [ "spack", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
@@ -94,18 +96,27 @@ spack:
- match:
- 'os=ubuntu18.04'
runner-attributes:
tags: ["spack", "public", "x86_64"]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: "default"
broken-specs-url: "s3://spack-binaries/broken-specs"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
service-job-attributes:
before_script:
- . "./share/spack/setup-env.sh"
- spack --version
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "x86_64"]
signing-job-attributes:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["spack", "aws"]
script:
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
cdash:
build-group: Build tests for different build systems
url: https://cdash.spack.io

25
share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
View File

@@ -42,7 +42,7 @@ spack:
+zfp
+visit
mirrors: { "mirror": "s3://spack-binaries/data-vis-sdk" }
mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" }
gitlab-ci:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -52,13 +52,15 @@ spack:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- spack -d ci rebuild
mappings:
- match:
- llvm
- qt
runner-attributes:
tags: [ "spack", "public", "huge", "x86_64" ]
tags: [ "spack", "huge", "x86_64" ]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
@@ -72,7 +74,7 @@ spack:
- visit
- vtk-m
runner-attributes:
tags: [ "spack", "public", "large", "x86_64" ]
tags: [ "spack", "large", "x86_64" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
@@ -98,7 +100,7 @@ spack:
- raja
- vtk-h
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
tags: [ "spack", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
@@ -133,7 +135,7 @@ spack:
- util-linux-uuid
runner-attributes:
tags: [ "spack", "public", "small", "x86_64" ]
tags: [ "spack", "small", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
@@ -141,11 +143,12 @@ spack:
- match: ['@:']
runner-attributes:
tags: ["spack", "public", "x86_64"]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: "default"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
broken-specs-url: "s3://spack-binaries/broken-specs"
service-job-attributes:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
before_script:
@@ -153,6 +156,14 @@ spack:
- spack --version
tags: ["spack", "public", "medium", "x86_64"]
signing-job-attributes:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["spack", "aws"]
script:
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
cdash:
build-group: Data and Vis SDK
url: https://cdash.spack.io

6
share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml
View File

@@ -32,7 +32,7 @@ spack:
- - $easy_specs
- - $arch
mirrors: { "mirror": "s3://spack-binaries/e4s-mac" }
mirrors: { "mirror": "s3://spack-binaries/develop/e4s-mac" }
gitlab-ci:
@@ -51,7 +51,9 @@ spack:
runner-attributes:
tags:
- omicron
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
broken-specs-url: "s3://spack-binaries/broken-specs"
service-job-attributes:
before_script:
- . "./share/spack/setup-env.sh"

25
share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
View File

@@ -222,7 +222,7 @@ spack:
- - $cuda_specs
- - $arch
mirrors: { "mirror": "s3://spack-binaries/e4s" }
mirrors: { "mirror": "s3://spack-binaries/develop/e4s" }
gitlab-ci:
@@ -233,6 +233,8 @@ spack:
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
@@ -240,7 +242,7 @@ spack:
- match:
- llvm
runner-attributes:
tags: [ "spack", "public", "huge", "x86_64" ]
tags: [ "spack", "huge", "x86_64" ]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
@@ -265,7 +267,7 @@ spack:
- vtk-m
- warpx
runner-attributes:
tags: [ "spack", "public", "large", "x86_64" ]
tags: [ "spack", "large", "x86_64" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
@@ -333,7 +335,7 @@ spack:
- vtk-h
- zfp
runner-attributes:
tags: [ "spack", "public", "medium", "x86_64" ]
tags: [ "spack", "medium", "x86_64" ]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
@@ -394,7 +396,7 @@ spack:
- zlib
- zstd
runner-attributes:
tags: [ "spack", "public", "small", "x86_64" ]
tags: [ "spack", "small", "x86_64" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
@@ -402,11 +404,12 @@ spack:
- match: ['os=ubuntu18.04']
runner-attributes:
tags: ["spack", "public", "x86_64"]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: "default"
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
broken-specs-url: "s3://spack-binaries/broken-specs"
service-job-attributes:
before_script:
- . "./share/spack/setup-env.sh"
@@ -414,6 +417,14 @@ spack:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "x86_64"]
signing-job-attributes:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["spack", "aws"]
script:
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
cdash:
build-group: New PR testing workflow
url: https://cdash.spack.io

22
share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
View File

@@ -54,7 +54,7 @@ spack:
- zfp
mirrors:
mirror: "s3://spack-binaries/radiuss"
mirror: "s3://spack-binaries/develop/radiuss"
specs:
- matrix:
@@ -69,6 +69,8 @@ spack:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- spack -d ci rebuild
mappings:
- match:
@@ -76,7 +78,7 @@ spack:
- openblas
- rust
runner-attributes:
tags: ["spack", "public", "large", "x86_64"]
tags: ["spack", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
@@ -96,7 +98,7 @@ spack:
- vtk-h
- vtk-m
runner-attributes:
tags: ["spack", "public", "medium", "x86_64"]
tags: ["spack", "medium", "x86_64"]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
@@ -150,7 +152,7 @@ spack:
- zfp
- zlib
runner-attributes:
tags: ["spack", "public", "small", "x86_64"]
tags: ["spack", "small", "x86_64"]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
@@ -158,10 +160,12 @@ spack:
- match: ['os=ubuntu18.04']
runner-attributes:
tags: ["spack", "public", "x86_64"]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: "default"
broken-specs-url: "s3://spack-binaries/broken-specs"
service-job-attributes:
before_script:
- . "./share/spack/setup-env.sh"
@@ -169,6 +173,14 @@ spack:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
tags: ["spack", "public", "x86_64"]
signing-job-attributes:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["spack", "aws"]
script:
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
cdash:
build-group: RADIUSS
url: https://cdash.spack.io

23
share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
View File

@@ -59,7 +59,7 @@ spack:
- $gcc_spack_built_packages
mirrors:
mirror: 's3://spack-binaries/tutorial'
mirror: 's3://spack-binaries/develop/tutorial'
gitlab-ci:
script:
@@ -69,6 +69,8 @@ spack:
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
- spack -d ci rebuild
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
@@ -81,7 +83,7 @@ spack:
- netlib-lapack
- trilinos
runner-attributes:
tags: ["spack", "public", "large", "x86_64"]
tags: ["spack", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
@@ -99,7 +101,7 @@ spack:
- py-scipy
- slurm
runner-attributes:
tags: ["spack", "public", "medium", "x86_64"]
tags: ["spack", "medium", "x86_64"]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
@@ -129,7 +131,7 @@ spack:
- tar
- util-linux-uuid
runner-attributes:
tags: ["spack", "public", "small", "x86_64"]
tags: ["spack", "small", "x86_64"]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
@@ -137,11 +139,12 @@ spack:
- match: ['@:']
runner-attributes:
tags: ["spack", "public", "x86_64"]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: default
broken-specs-url: "s3://spack-binaries-develop/broken-specs"
broken-specs-url: "s3://spack-binaries/broken-specs"
service-job-attributes:
image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] }
before_script:
@@ -149,6 +152,14 @@ spack:
- spack --version
tags: ["spack", "public", "x86_64"]
signing-job-attributes:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["spack", "aws"]
script:
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
cdash:
build-group: Spack Tutorial
url: https://cdash.spack.io