zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first set of work to allow for saving local results with spack monitor (#23804)

Browse Source 
This work will come in two phases. The first here is to allow saving of a local result
with spack monitor, and the second will add a spack monitor command so the user can
do spack monitor upload.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Co-authored-by: vsoch <vsoch@users.noreply.github.com>
This commit is contained in:
Vanessasaurus 2021-05-25 12:29:34 -06:00 committed by GitHub
parent e22da8df05
commit b44bb952eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 155 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/spack/docs/monitoring.rst
View File

@ -102,3 +102,19 @@ more tags to your build, you can do:
# Add two tags, "pizza" and "pasta"
$ spack install --monitor --monitor-tags pizza,pasta hdf5
------------------
Monitoring Offline
------------------
In the case that you want to save monitor results to your filesystem
and then upload them later (perhaps you are in an environment where you don't
have credentials or it isn't safe to use them) you can use the ``--monitor-save-local``
flag.
.. code-block:: console
$ spack install --monitor --monitor-save-local hdf5
This will save results in a subfolder, "monitor" in your designated spack
reports folder, which defaults to ``$HOME/.spack/reports/monitor``.

1
lib/spack/spack/cmd/install.py
View File

@ -306,6 +306,7 @@ def install(parser, args, **kwargs):
prefix=args.monitor_prefix,
disable_auth=args.monitor_disable_auth,
tags=args.monitor_tags,
save_local=args.monitor_save_local,
)
reporter = spack.report.collect_info(

113
lib/spack/spack/monitor.py
View File

@ -7,6 +7,8 @@
https://github.com/spack/spack-monitor/blob/main/script/spackmoncli.py
"""
from datetime import datetime
import hashlib
import base64
import os
import re
@ -18,11 +20,13 @@
from urllib2 import urlopen, Request, URLError # type: ignore # novm
import spack
import spack.config
import spack.hash_types as ht
import spack.main
import spack.store
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
import spack.util.path
import llnl.util.tty as tty
from copy import deepcopy
@ -31,7 +35,8 @@
cli = None
def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None):
def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None,
save_local=False):
"""
Get a monitor client for a particular host and prefix.
@ -47,26 +52,25 @@ def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=No
"""
global cli
cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail,
tags=tags)
tags=tags, save_local=save_local)
# If we don't disable auth, environment credentials are required
if not disable_auth:
if not disable_auth and not save_local:
cli.require_auth()
# We will exit early if the monitoring service is not running
info = cli.service_info()
# We will exit early if the monitoring service is not running, but
# only if we aren't doing a local save
if not save_local:
info = cli.service_info()
# If we allow failure, the response will be done
if info:
tty.debug("%s v.%s has status %s" % (
info['id'],
info['version'],
info['status'])
)
return cli
else:
tty.debug("spack-monitor server not found, continuing as allow_fail is True.")
# If we allow failure, the response will be done
if info:
tty.debug("%s v.%s has status %s" % (
info['id'],
info['version'],
info['status'])
)
return cli
def get_monitor_group(subparser):
@ -82,6 +86,9 @@ def get_monitor_group(subparser):
monitor_group.add_argument(
'--monitor', action='store_true', dest='use_monitor', default=False,
help="interact with a montor server during builds.")
monitor_group.add_argument(
'--monitor-save-local', action='store_true', dest='monitor_save_local',
default=False, help="save monitor results to .spack instead of server.")
monitor_group.add_argument(
'--monitor-no-auth', action='store_true', dest='monitor_disable_auth',
default=False, help="the monitoring server does not require auth.")
@ -110,7 +117,8 @@ class SpackMonitorClient:
to the client on init.
"""
def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None):
def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None,
save_local=False):
self.host = host or "http://127.0.0.1"
self.baseurl = "%s/%s" % (self.host, prefix.strip("/"))
self.token = os.environ.get("SPACKMON_TOKEN")
@ -120,9 +128,34 @@ def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None):
self.spack_version = spack.main.get_version()
self.capture_build_environment()
self.tags = tags
self.save_local = save_local
# We keey lookup of build_id by full_hash
self.build_ids = {}
self.setup_save()
def setup_save(self):
"""Given a local save "save_local" ensure the output directory exists.
"""
if not self.save_local:
return
save_dir = spack.util.path.canonicalize_path(
spack.config.get('config:monitor_dir', '~/.spack/reports/monitor'))
# Name based on timestamp
now = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%s')
self.save_dir = os.path.join(save_dir, now)
if not os.path.exists(self.save_dir):
os.makedirs(self.save_dir)
def save(self, obj, filename):
"""
Save a monitor json result to the save directory.
"""
filename = os.path.join(self.save_dir, filename)
write_json(obj, filename)
return {"message": "Build saved locally to %s" % filename}
def load_build_environment(self, spec):
"""
@ -174,7 +207,7 @@ def require_auth(self):
The token and username must not be unset
"""
if not self.token or not self.username:
if not self.save_local and (not self.token or not self.username):
tty.die("You are required to export SPACKMON_TOKEN and SPACKMON_USER")
def set_header(self, name, value):
@ -346,8 +379,14 @@ def new_configuration(self, specs):
spec.concretize()
as_dict = {"spec": spec.to_dict(hash=ht.full_hash),
"spack_version": self.spack_version}
response = self.do_request("specs/new/", data=sjson.dump(as_dict))
configs[spec.package.name] = response.get('data', {})
if self.save_local:
filename = "spec-%s-%s-config.json" % (spec.name, spec.version)
self.save(sjson.dump(as_dict), filename)
else:
response = self.do_request("specs/new/", data=sjson.dump(as_dict))
configs[spec.package.name] = response.get('data', {})
return configs
def new_build(self, spec):
@ -384,6 +423,27 @@ def get_build_id(self, spec, return_response=False, spec_exists=True):
spec_file = os.path.join(meta_dir, "spec.yaml")
data['spec'] = syaml.load(read_file(spec_file))
if self.save_local:
return self.get_local_build_id(data, full_hash, return_response)
return self.get_server_build_id(data, full_hash, return_response)
def get_local_build_id(self, data, full_hash, return_response):
"""
Generate a local build id based on hashing the expected data
"""
hasher = hashlib.md5()
hasher.update(str(data).encode('utf-8'))
bid = hasher.hexdigest()
filename = "build-metadata-%s.json" % full_hash
response = self.save(sjson.dump(data), filename)
if return_response:
return response
return bid
def get_server_build_id(self, data, full_hash, return_response=False):
"""
Retrieve a build id from the spack monitor server
"""
response = self.do_request("builds/new/", data=sjson.dump(data))
# Add the build id to the lookup
@ -403,6 +463,10 @@ def update_build(self, spec, status="SUCCESS"):
successful install. This endpoint can take a general status to update.
"""
data = {"build_id": self.get_build_id(spec), "status": status}
if self.save_local:
filename = "build-%s-status.json" % data['build_id']
return self.save(sjson.dump(data), filename)
return self.do_request("builds/update/", data=sjson.dump(data))
def fail_task(self, spec):
@ -444,6 +508,10 @@ def send_phase(self, pkg, phase_name, phase_output_file, status):
"output": read_file(phase_output_file),
"phase_name": phase_name})
if self.save_local:
filename = "build-%s-phase-%s.json" % (data['build_id'], phase_name)
return self.save(sjson.dump(data), filename)
return self.do_request("builds/phases/update/", data=sjson.dump(data))
def upload_specfile(self, filename):
@ -459,6 +527,11 @@ def upload_specfile(self, filename):
# We load as json just to validate it
spec = read_json(filename)
data = {"spec": spec, "spack_verison": self.spack_version}
if self.save_local:
filename = "spec-%s-%s.json" % (spec.name, spec.version)
return self.save(sjson.dump(data), filename)
return self.do_request("specs/new/", data=sjson.dump(data))

2
lib/spack/spack/paths.py
View File

@ -54,7 +54,7 @@
user_bootstrap_path = os.path.join(user_config_path, 'bootstrap')
user_bootstrap_store = os.path.join(user_bootstrap_path, 'store')
reports_path = os.path.join(user_config_path, "reports")
monitor_path = os.path.join(reports_path, "monitor")
opt_path = os.path.join(prefix, "opt")
etc_path = os.path.join(prefix, "etc")

42
lib/spack/spack/test/monitor.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import spack.config
import spack.spec
from spack.main import SpackCommand
import pytest
import os
install = SpackCommand('install')
@pytest.fixture(scope='session')
def test_install_monitor_save_local(install_mockery_mutable_config,
mock_fetch, tmpdir_factory):
"""
Mock installing and saving monitor results to file.
"""
reports_dir = tmpdir_factory.mktemp('reports')
spack.config.set('config:monitor_dir', str(reports_dir))
out = install('--monitor', '--monitor-save-local', 'dttop')
assert "Successfully installed dttop" in out
# The reports directory should not be empty (timestamped folders)
assert os.listdir(str(reports_dir))
# Get the spec name
spec = spack.spec.Spec("dttop")
spec.concretize()
full_hash = spec.full_hash()
# Ensure we have monitor results saved
for dirname in os.listdir(str(reports_dir)):
dated_dir = os.path.join(str(reports_dir), dirname)
build_metadata = "build-metadata-%s.json" % full_hash
assert build_metadata in os.listdir(dated_dir)
spec_file = "spec-dttop-%s-config.json" % spec.version
assert spec_file in os.listdir(dated_dir)
spack.config.set('config:monitor_dir', "~/.spack/reports/monitor")

4
share/spack/spack-completion.bash
View File

@ -358,7 +358,7 @@ _spack_add() {
_spack_analyze() {
if $list_options
then
SPACK_COMPREPLY="-h --help --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix"
SPACK_COMPREPLY="-h --help --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix"
else
SPACK_COMPREPLY="list-analyzers run"
fi
@ -1063,7 +1063,7 @@ _spack_info() {
_spack_install() {
if $list_options
then
SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all"
SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all"
else
_all_packages
fi