85e13260cf
This PR supports the creation of securely signed binaries built from spack
develop as well as release branches and tags. Specifically:
- remove internal pr mirror url generation logic in favor of buildcache destination
on command line
- with a single mirror url specified in the spack.yaml, this makes it clearer where
binaries from various pipelines are pushed
- designate some tags as reserved: ['public', 'protected', 'notary']
- these tags are stripped from all jobs by default and provisioned internally
based on pipeline type
- update gitlab ci yaml to include pipelines on more protected branches than just
develop (so include releases and tags)
- binaries from all protected pipelines are pushed into mirrors including the
branch name so releases, tags, and develop binaries are kept separate
- update rebuild jobs running on protected pipelines to run on special runners
provisioned with an intermediate signing key
- protected rebuild jobs no longer use "SPACK_SIGNING_KEY" env var to
obtain signing key (in fact, final signing key is nowhere available to rebuild jobs)
- these intermediate signatures are verified at the end of each pipeline by a new
signing job to ensure binaries were produced by a protected pipeline
- optionallly schedule a signing/notary job at the end of the pipeline to sign all
packges in the mirror
- add signing-job-attributes to gitlab-ci section of spack environment to allow
configuration
- signing job runs on special runner (separate from protected rebuild runners)
provisioned with public intermediate key and secret signing key
125 lines
3.5 KiB
YAML
125 lines
3.5 KiB
YAML
spack:
|
|
view: false
|
|
|
|
concretizer:
|
|
reuse: false
|
|
unify: when_possible
|
|
|
|
config:
|
|
install_tree:
|
|
root: /home/software/spack
|
|
padded_length: 512
|
|
projections:
|
|
all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
|
|
|
|
definitions:
|
|
- default_specs:
|
|
- lz4 # MakefilePackage
|
|
- mpich~fortran # AutotoolsPackage
|
|
- tut # WafPackage
|
|
- py-setuptools # PythonPackage
|
|
- openjpeg # CMakePackage
|
|
- r-rcpp # RPackage
|
|
- ruby-rake # RubyPackage
|
|
- arch:
|
|
- '%gcc target=x86_64'
|
|
|
|
specs:
|
|
- matrix:
|
|
- - $default_specs
|
|
- - $arch
|
|
|
|
mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" }
|
|
|
|
gitlab-ci:
|
|
script:
|
|
- . "./share/spack/setup-env.sh"
|
|
- spack --version
|
|
- cd ${SPACK_CONCRETE_ENV_DIR}
|
|
- spack env activate --without-view .
|
|
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
|
|
- if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
|
|
- if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi
|
|
- spack -d ci rebuild
|
|
|
|
image:
|
|
name: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
|
|
entrypoint: [ "" ]
|
|
|
|
mappings:
|
|
- match:
|
|
- cmake
|
|
runner-attributes:
|
|
tags: [ "spack", "large", "x86_64"]
|
|
variables:
|
|
CI_JOB_SIZE: large
|
|
KUBERNETES_CPU_REQUEST: 8000m
|
|
KUBERNETES_MEMORY_REQUEST: 12G
|
|
|
|
- match:
|
|
- curl
|
|
- gettext
|
|
- mpich
|
|
- openjpeg
|
|
- sqlite
|
|
runner-attributes:
|
|
tags: [ "spack", "medium", "x86_64" ]
|
|
variables:
|
|
CI_JOB_SIZE: "medium"
|
|
KUBERNETES_CPU_REQUEST: "2000m"
|
|
KUBERNETES_MEMORY_REQUEST: "4G"
|
|
|
|
- match:
|
|
- bzip2
|
|
- diffutils
|
|
- findutils
|
|
- libffi
|
|
- libidn2
|
|
- libmd
|
|
- libsigsegv
|
|
- libxml2
|
|
- lz4
|
|
- openssl
|
|
- pkgconf
|
|
- tut
|
|
- util-linux-uuid
|
|
- util-macros
|
|
- xz
|
|
- zlib
|
|
runner-attributes:
|
|
tags: [ "spack", "medium", "x86_64" ]
|
|
variables:
|
|
CI_JOB_SIZE: "small"
|
|
KUBERNETES_CPU_REQUEST: "500m"
|
|
KUBERNETES_MEMORY_REQUEST: "500M"
|
|
|
|
- match:
|
|
- 'os=ubuntu18.04'
|
|
runner-attributes:
|
|
tags: ["spack", "x86_64"]
|
|
variables:
|
|
CI_JOB_SIZE: "default"
|
|
|
|
broken-specs-url: "s3://spack-binaries/broken-specs"
|
|
|
|
service-job-attributes:
|
|
before_script:
|
|
- . "./share/spack/setup-env.sh"
|
|
- spack --version
|
|
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
|
|
tags: ["spack", "public", "x86_64"]
|
|
|
|
signing-job-attributes:
|
|
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
|
|
tags: ["spack", "aws"]
|
|
script:
|
|
- aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
|
|
- /sign.sh
|
|
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
|
|
|
|
cdash:
|
|
build-group: Build tests for different build systems
|
|
url: https://cdash.spack.io
|
|
project: Spack Testing
|
|
site: Cloud Gitlab Infrastructure
|