zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor release-jobs cmd based on use of environments (no docker either)

Browse Source
This commit is contained in:
Scott Wittenburg
2019-05-28 17:21:55 -06:00
committed by Todd Gamblin
parent 73c161b7de
commit fabbb3d58a
4 changed files with 198 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

424
lib/spack/spack/cmd/release_jobs.py
View File

@@ -3,26 +3,20 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import json
import os
import shutil
import tempfile
import sys
import subprocess
from jsonschema import validate, ValidationError
from six import iteritems
from six.moves.urllib.request import build_opener, HTTPHandler, Request
from six.moves.urllib.parse import urlencode
import llnl.util.tty as tty
from spack.architecture import sys_type
import spack.environment as ev
from spack.dependency import all_deptypes
from spack.spec import Spec, CompilerSpec
from spack.paths import spack_root
from spack.error import SpackError
from spack.schema.os_container_mapping import schema as mapping_schema
from spack.spec import Spec
from spack.schema.specs_deps import schema as specs_deps_schema
from spack.spec_set import CombinatorialSpecSet
import spack.util.spack_yaml as syaml
description = "generate release build set as .gitlab-ci.yml"
@@ -32,57 +26,101 @@
def setup_parser(subparser):
subparser.add_argument(
'-s', '--spec-set', default=None,
help="path to release spec-set yaml file")
subparser.add_argument(
'-m', '--mirror-url', default=None,
help="url of binary mirror where builds should be pushed")
'-f', '--force', action='store_true', default=False,
help="Force re-concretization of environment first")
subparser.add_argument(
'-o', '--output-file', default=".gitlab-ci.yml",
help="path to output file to write")
subparser.add_argument(
'-t', '--shared-runner-tag', default=None,
help="tag to add to jobs for shared runner selection")
subparser.add_argument(
'-k', '--signing-key', default=None,
help="hash of gpg key to use for package signing")
subparser.add_argument(
'-c', '--cdash-url', default='https://cdash.spack.io',
help="Base url of CDash instance jobs should communicate with")
subparser.add_argument(
'-p', '--print-summary', action='store_true', default=False,
help="Print summary of staged jobs to standard output")
subparser.add_argument(
'--resolve-deps-locally', action='store_true', default=False,
help="Use only the current machine to concretize specs, " +
"instead of iterating over items in os-container-mapping.yaml " +
"and using docker run. Assumes the current machine architecure " +
"is listed in the os-container-mapping.yaml config file.")
subparser.add_argument(
'--specs-deps-output', default='/dev/stdout',
help="A file path to which spec deps should be written. This " +
"argument is generally for internal use, and should not be " +
"provided by end-users under normal conditions.")
subparser.add_argument(
'specs', nargs=argparse.REMAINDER,
help="These positional arguments are generally for internal use. " +
"The --spec-set argument should be used to identify a yaml " +
"file describing the set of release specs to include in the " +
".gitlab-ci.yml file.")
'-c', '--cdash-credentials', default=None,
help="Path to file containing CDash authentication token")
def get_job_name(spec, osarch):
return '{0} {1} {2} {3}'.format(spec.name, spec.version,
spec.compiler, osarch)
def _create_buildgroup(opener, headers, url, project, group_name, group_type):
data = {
"newbuildgroup": group_name,
"project": project,
"type": group_type
}
request = Request(url, data=json.dumps(data), headers=headers)
response = opener.open(request)
response_code = response.getcode()
if response_code != 200 and response_code != 201:
print('Creating buildgroup failed (response code = {0}'.format(
response_code))
return None
response_text = response.read()
response_json = json.loads(response_text)
build_group_id = response_json['id']
return build_group_id
def populate_buildgroup(job_names, group_name, project, site,
credentials, cdash_url, exit_on_fail=False):
url = "{0}/api/v1/buildgroup.php".format(cdash_url)
headers = {
'Authorization': 'Bearer {0}'.format(credentials),
'Content-Type': 'application/json',
}
opener = build_opener(HTTPHandler)
parent_group_id = _create_buildgroup(
opener, headers, url, project, group_name, 'Daily')
group_id = _create_buildgroup(
opener, headers, url, project, 'Latest {0}'.format(group_name),
'Latest')
if not parent_group_id or not group_id:
print('Unable to create or retrieve the build groups')
sys.exit(1)
data = {
'project': project,
'buildgroupid': group_id,
'dynamiclist': [{
'match': name,
'parentgroupid': parent_group_id,
'site': site
} for name in job_names]
}
request = Request(url, data=json.dumps(data), headers=headers)
request.get_method = lambda: 'PUT'
response = opener.open(request)
response_code = response.getcode()
if response_code != 200 and exit_on_fail:
print('Unexpected response ({0}) when populating buildgroup'.format(
response_code))
sys.exit(1)
def get_job_name(spec, osarch, build_group):
return '{0} {1} {2} {3} {4}'.format(
spec.name, spec.version, spec.compiler, osarch, build_group)
def get_cdash_build_name(spec, build_group):
return '{0}@{1}%{2} arch={3} ({4})'.format(
spec.name, spec.version, spec.compiler, spec.architecture, build_group)
def get_spec_string(spec):
@@ -109,84 +147,8 @@ def _add_dependency(spec_label, dep_label, deps):
deps[spec_label].add(dep_label)
def get_deps_using_container(specs, image):
image_home_dir = '/home/spackuser'
repo_mount_location = '{0}/spack'.format(image_home_dir)
temp_dir = tempfile.mkdtemp(dir='/tmp')
# The paths this module will see (from outside the container)
temp_file = os.path.join(temp_dir, 'spec_deps.json')
temp_err = os.path.join(temp_dir, 'std_err.log')
# The paths the bash_command will see inside the container
json_output = '/work/spec_deps.json'
std_error = '/work/std_err.log'
specs_arg = ' '.join([str(spec) for spec in specs])
bash_command = " ".join(["source {0}/share/spack/setup-env.sh ;",
"spack release-jobs",
"--specs-deps-output {1}",
"{2}",
"2> {3}"]).format(
repo_mount_location, json_output, specs_arg, std_error)
docker_cmd_to_run = [
'docker', 'run', '--rm',
'-v', '{0}:{1}'.format(spack_root, repo_mount_location),
'-v', '{0}:{1}'.format(temp_dir, '/work'),
'--entrypoint', 'bash',
'-t', str(image),
'-c',
bash_command,
]
tty.debug('Running subprocess command:')
tty.debug(' '.join(docker_cmd_to_run))
# Docker is going to merge the stdout/stderr from the script and write it
# all to the stdout of the running container. For this reason, we won't
# pipe any stdout/stderr from the docker command, but rather write the
# output we care about to a file in a mounted directory. Similarly, any
# errors from running the spack command inside the container are redirected
# to another file in the mounted directory.
proc = subprocess.Popen(docker_cmd_to_run)
proc.wait()
# Check for errors from spack command
if os.path.exists(temp_err) and os.path.getsize(temp_err) > 0:
# Spack wrote something to stderr inside the container. We will
# print out whatever it is, but attempt to carry on with the process.
tty.error('Encountered spack error running command in container:')
with open(temp_err, 'r') as err:
tty.error(err.read())
spec_deps_obj = {}
try:
# Finally, try to read/parse the output we really care about: the
# specs and dependency edges for the provided spec, as it was
# concretized in the appropriate container.
with open(temp_file, 'r') as fd:
spec_deps_obj = json.loads(fd.read())
except ValueError as val_err:
tty.error('Failed to read json object from spec-deps output file:')
tty.error(str(val_err))
except IOError as io_err:
tty.error('Problem reading from spec-deps json output file:')
tty.error(str(io_err))
finally:
shutil.rmtree(temp_dir)
return spec_deps_obj
def get_spec_dependencies(specs, deps, spec_labels, image=None):
if image:
spec_deps_obj = get_deps_using_container(specs, image)
else:
spec_deps_obj = compute_spec_deps(specs)
def get_spec_dependencies(specs, deps, spec_labels):
spec_deps_obj = compute_spec_deps(specs)
try:
validate(spec_deps_obj, specs_deps_schema)
@@ -210,7 +172,7 @@ def get_spec_dependencies(specs, deps, spec_labels, image=None):
_add_dependency(entry['spec'], entry['depends'], deps)
def stage_spec_jobs(spec_set, containers, current_system=None):
def stage_spec_jobs(specs):
"""Take a set of release specs along with a dictionary describing the
available docker containers and what compilers they have, and generate
a list of "stages", where the jobs in any stage are dependent only on
@@ -265,46 +227,7 @@ def remove_satisfied_deps(deps, satisfied_list):
deps = {}
spec_labels = {}
if current_system:
if current_system not in containers:
error_msg = ' '.join(['Current system ({0}) does not appear in',
'os_container_mapping.yaml, ignoring',
'request']).format(
current_system)
raise SpackError(error_msg)
os_names = [current_system]
else:
os_names = [name for name in containers]
container_specs = {}
for name in os_names:
container_specs[name] = {'image': None, 'specs': []}
# Collect together all the specs that should be concretized in each
# container so they can all be done at once, avoiding the need to
# run the docker container for each spec separately.
for spec in spec_set:
for osname in os_names:
container_info = containers[osname]
image = None if current_system else container_info['image']
if image:
container_specs[osname]['image'] = image
if 'compilers' in container_info:
found_at_least_one = False
for item in container_info['compilers']:
container_compiler_spec = CompilerSpec(item['name'])
if spec.compiler == container_compiler_spec:
container_specs[osname]['specs'].append(spec)
found_at_least_one = True
if not found_at_least_one:
tty.warn('No compiler in {0} satisfied {1}'.format(
osname, spec.compiler))
for osname in container_specs:
if container_specs[osname]['specs']:
image = container_specs[osname]['image']
specs = container_specs[osname]['specs']
get_spec_dependencies(specs, deps, spec_labels, image)
get_spec_dependencies(specs, deps, spec_labels)
# Save the original deps, as we need to return them at the end of the
# function. In the while loop below, the "dependencies" variable is
@@ -452,49 +375,48 @@ def append_dep(s, d):
return deps_json_obj
def spec_matches(spec, match_string):
return spec.satisfies(match_string)
def find_matching_config(spec, ci_mappings):
for ci_mapping in ci_mappings:
for match_string in ci_mapping['match']:
if spec_matches(spec, match_string):
return ci_mapping['runner-attributes']
return None
def release_jobs(parser, args):
share_path = os.path.join(spack_root, 'share', 'spack', 'docker')
os_container_mapping_path = os.path.join(
share_path, 'os-container-mapping.yaml')
env = ev.get_env(args, 'release-jobs', required=True)
env.concretize(force=args.force)
with open(os_container_mapping_path, 'r') as fin:
os_container_mapping = syaml.load(fin)
yaml_root = env.yaml['spack']
try:
validate(os_container_mapping, mapping_schema)
except ValidationError as val_err:
tty.error('Ill-formed os-container-mapping configuration object')
tty.error(os_container_mapping)
tty.debug(val_err)
return
if 'gitlab-ci' not in yaml_root:
tty.die('Environment yaml does not have "gitlab-ci" section')
containers = os_container_mapping['containers']
ci_mappings = yaml_root['gitlab-ci']['mappings']
if args.specs:
# Just print out the spec labels and all dependency edges in
# a json format.
spec_list = [Spec(s) for s in args.specs]
with open(args.specs_deps_output, 'w') as out:
compute_spec_deps(spec_list, out)
return
ci_cdash = yaml_root['cdash']
build_group = ci_cdash['build-group']
cdash_url = ci_cdash['url']
cdash_project = ci_cdash['project']
proj_enc = urlencode({'project': cdash_project})
eq_idx = proj_enc.find('=') + 1
cdash_project_enc = proj_enc[eq_idx:]
cdash_site = ci_cdash['site']
cdash_auth_token = None
current_system = sys_type() if args.resolve_deps_locally else None
if args.cdash_credentials:
with open(args.cdash_credentials) as fd:
cdash_auth_token = fd.read()
cdash_auth_token = cdash_auth_token.strip()
release_specs_path = args.spec_set
if not release_specs_path:
raise SpackError('Must provide path to release spec-set')
ci_mirrors = yaml_root['mirrors']
mirror_urls = ci_mirrors.values()
release_spec_set = CombinatorialSpecSet.from_file(release_specs_path)
mirror_url = args.mirror_url
if not mirror_url:
raise SpackError('Must provide url of target binary mirror')
cdash_url = args.cdash_url
spec_labels, dependencies, stages = stage_spec_jobs(
release_spec_set, containers, current_system)
spec_labels, dependencies, stages = stage_spec_jobs(env.all_specs())
if not stages:
tty.msg('No jobs staged, exiting.')
@@ -503,6 +425,7 @@ def release_jobs(parser, args):
if args.print_summary:
print_staging_summary(spec_labels, dependencies, stages)
all_job_names = []
output_object = {}
job_count = 0
@@ -516,37 +439,56 @@ def release_jobs(parser, args):
release_spec = spec_labels[spec_label]['spec']
root_spec = spec_labels[spec_label]['rootSpec']
pkg_compiler = release_spec.compiler
pkg_hash = release_spec.dag_hash()
runner_attribs = find_matching_config(release_spec, ci_mappings)
if not runner_attribs:
tty.warn('No match found for {0}, skipping it'.format(
release_spec))
continue
tags = [tag for tag in runner_attribs['tags']]
variables = {}
if 'variables' in runner_attribs:
variables.update(runner_attribs['variables'])
build_image = None
if 'image' in runner_attribs:
build_image = runner_attribs['image']
osname = str(release_spec.architecture)
job_name = get_job_name(release_spec, osname)
container_info = containers[osname]
build_image = container_info['image']
job_name = get_job_name(release_spec, osname, build_group)
cdash_build_name = get_cdash_build_name(release_spec, build_group)
all_job_names.append(cdash_build_name)
job_scripts = ['./bin/rebuild-package.sh']
if 'setup_script' in container_info:
job_scripts.insert(
0, container_info['setup_script'] % pkg_compiler)
job_dependencies = []
if spec_label in dependencies:
job_dependencies = (
[get_job_name(spec_labels[dep_label]['spec'], osname)
for dep_label in dependencies[spec_label]])
[get_job_name(spec_labels[d]['spec'], osname, build_group)
for d in dependencies[spec_label]])
job_variables = {
'MIRROR_URL': mirror_urls[0],
'CDASH_BASE_URL': cdash_url,
'CDASH_PROJECT': cdash_project,
'CDASH_PROJECT_ENC': cdash_project_enc,
'CDASH_BUILD_NAME': cdash_build_name,
'DEPENDENCIES': ';'.join(job_dependencies),
'ROOT_SPEC': str(root_spec),
}
if args.signing_key:
job_variables['SIGN_KEY_HASH'] = args.signing_key
variables.update(job_variables)
job_object = {
'stage': stage_name,
'variables': {
'MIRROR_URL': mirror_url,
'CDASH_BASE_URL': cdash_url,
'HASH': pkg_hash,
'DEPENDENCIES': ';'.join(job_dependencies),
'ROOT_SPEC': str(root_spec),
},
'variables': variables,
'script': job_scripts,
'image': build_image,
'artifacts': {
'paths': [
'local_mirror/build_cache',
@@ -556,51 +498,41 @@ def release_jobs(parser, args):
'when': 'always',
},
'dependencies': job_dependencies,
'tags': tags,
}
# If we see 'compilers' in the container iformation, it's a
# filter for the compilers this container can handle, else we
# assume it can handle any compiler
if 'compilers' in container_info:
do_job = False
for item in container_info['compilers']:
container_compiler_spec = CompilerSpec(item['name'])
if pkg_compiler == container_compiler_spec:
do_job = True
else:
do_job = True
if build_image:
job_object['image'] = build_image
if args.shared_runner_tag:
job_object['tags'] = [args.shared_runner_tag]
if args.signing_key:
job_object['variables']['SIGN_KEY_HASH'] = args.signing_key
if do_job:
output_object[job_name] = job_object
job_count += 1
output_object[job_name] = job_object
job_count += 1
stage += 1
tty.msg('{0} build jobs generated in {1} stages'.format(
job_count, len(stages)))
final_stage = 'stage-rebuild-index'
# Use "all_job_names" to populate the build group for this set
if cdash_auth_token:
populate_buildgroup(all_job_names, build_group, cdash_project,
cdash_site, cdash_auth_token, cdash_url)
else:
tty.warn('Unable to populate buildgroup without CDash credentials')
# Add an extra, final job to regenerate the index
final_stage = 'stage-rebuild-index'
final_job = {
'stage': final_stage,
'variables': {
'MIRROR_URL': mirror_url,
'MIRROR_URL': mirror_urls[0],
},
'image': build_image,
'image': 'scottwittenburg/spack_ci_generator_alpine', # just needs some basic python image
'script': './bin/rebuild-index.sh',
'tags': ['spack-k8s'] # may want a runner to handle this
}
if args.shared_runner_tag:
final_job['tags'] = [args.shared_runner_tag]
output_object['rebuild-index'] = final_job
stage_names.append(final_stage)
output_object['stages'] = stage_names
with open(args.output_file, 'w') as outf:

50
lib/spack/spack/schema/os_container_mapping.py
View File

@@ -1,50 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Schema for os-container-mapping.yaml configuration file.
.. literalinclude:: _spack_root/lib/spack/spack/schema/os_container_mapping.py
:lines: 32-
"""
schema = {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack release builds os/container mapping config file schema',
'type': 'object',
'additionalProperties': False,
'patternProperties': {
r'containers': {
'type': 'object',
'default': {},
'patternProperties': {
r'[\w\d\-_\.]+': {
'type': 'object',
'default': {},
'additionalProperties': False,
'required': ['image'],
'properties': {
'image': {'type': 'string'},
'setup_script': {'type': 'string'},
'compilers': {
'type': 'array',
'default': [],
'items': {
'type': 'object',
'default': {},
'additionalProperties': False,
'required': ['name'],
'properties': {
'name': {'type': 'string'},
'path': {'type': 'string'},
},
},
},
},
},
},
},
},
}