first set of work to allow for saving local results with spack monitor (#23804)

This work will come in two phases. The first here is to allow saving of a local result
with spack monitor, and the second will add a spack monitor command so the user can
do spack monitor upload.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Co-authored-by: vsoch <vsoch@users.noreply.github.com>

lib/spack/docs/monitoring.rst

@@ -102,3 +102,19 @@ more tags to your build, you can do:
     # Add two tags, "pizza" and "pasta"
     $ spack install --monitor --monitor-tags pizza,pasta hdf5
 
+
+------------------
+Monitoring Offline
+------------------
+
+In the case that you want to save monitor results to your filesystem
+and then upload them later (perhaps you are in an environment where you don't
+have credentials or it isn't safe to use them) you can use the ``--monitor-save-local``
+flag.
+
+.. code-block:: console
+
+    $ spack install --monitor --monitor-save-local hdf5 
+
+This will save results in a subfolder, "monitor" in your designated spack
+reports folder, which defaults to ``$HOME/.spack/reports/monitor``.

lib/spack/spack/cmd/install.py

@@ -306,6 +306,7 @@ def install(parser, args, **kwargs):
             prefix=args.monitor_prefix,
             disable_auth=args.monitor_disable_auth,
             tags=args.monitor_tags,
+            save_local=args.monitor_save_local,
         )
 
     reporter = spack.report.collect_info(

lib/spack/spack/monitor.py

@@ -7,6 +7,8 @@
 https://github.com/spack/spack-monitor/blob/main/script/spackmoncli.py
 """
 
+from datetime import datetime
+import hashlib
 import base64
 import os
 import re
@@ -18,11 +20,13 @@
     from urllib2 import urlopen, Request, URLError  # type: ignore  # novm
 
 import spack
+import spack.config
 import spack.hash_types as ht
 import spack.main
 import spack.store
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
+import spack.util.path
 import llnl.util.tty as tty
 from copy import deepcopy
 
@@ -31,7 +35,8 @@
 cli = None
 
 
-def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None):
+def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None,
+               save_local=False):
     """
     Get a monitor client for a particular host and prefix.
 
@@ -47,13 +52,15 @@ def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=No
     """
     global cli
     cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail,
-                             tags=tags)
+                             tags=tags, save_local=save_local)
 
     # If we don't disable auth, environment credentials are required
-    if not disable_auth:
+    if not disable_auth and not save_local:
         cli.require_auth()
 
-    # We will exit early if the monitoring service is not running
+    # We will exit early if the monitoring service is not running, but
+    # only if we aren't doing a local save
+    if not save_local:
         info = cli.service_info()
 
         # If we allow failure, the response will be done
@@ -65,9 +72,6 @@ def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=No
             )
     return cli
 
-    else:
-        tty.debug("spack-monitor server not found, continuing as allow_fail is True.")
-
 
 def get_monitor_group(subparser):
     """
@@ -82,6 +86,9 @@ def get_monitor_group(subparser):
     monitor_group.add_argument(
         '--monitor', action='store_true', dest='use_monitor', default=False,
         help="interact with a montor server during builds.")
+    monitor_group.add_argument(
+        '--monitor-save-local', action='store_true', dest='monitor_save_local',
+        default=False, help="save monitor results to .spack instead of server.")
     monitor_group.add_argument(
         '--monitor-no-auth', action='store_true', dest='monitor_disable_auth',
         default=False, help="the monitoring server does not require auth.")
@@ -110,7 +117,8 @@ class SpackMonitorClient:
     to the client on init.
     """
 
-    def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None):
+    def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None,
+                 save_local=False):
         self.host = host or "http://127.0.0.1"
         self.baseurl = "%s/%s" % (self.host, prefix.strip("/"))
         self.token = os.environ.get("SPACKMON_TOKEN")
@@ -120,9 +128,34 @@ def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None):
         self.spack_version = spack.main.get_version()
         self.capture_build_environment()
         self.tags = tags
+        self.save_local = save_local
 
         # We keey lookup of build_id by full_hash
         self.build_ids = {}
+        self.setup_save()
+
+    def setup_save(self):
+        """Given a local save "save_local" ensure the output directory exists.
+        """
+        if not self.save_local:
+            return
+
+        save_dir = spack.util.path.canonicalize_path(
+            spack.config.get('config:monitor_dir', '~/.spack/reports/monitor'))
+
+        # Name based on timestamp
+        now = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%s')
+        self.save_dir = os.path.join(save_dir, now)
+        if not os.path.exists(self.save_dir):
+            os.makedirs(self.save_dir)
+
+    def save(self, obj, filename):
+        """
+        Save a monitor json result to the save directory.
+        """
+        filename = os.path.join(self.save_dir, filename)
+        write_json(obj, filename)
+        return {"message": "Build saved locally to %s" % filename}
 
     def load_build_environment(self, spec):
         """
@@ -174,7 +207,7 @@ def require_auth(self):
 
         The token and username must not be unset
         """
-        if not self.token or not self.username:
+        if not self.save_local and (not self.token or not self.username):
             tty.die("You are required to export SPACKMON_TOKEN and SPACKMON_USER")
 
     def set_header(self, name, value):
@@ -346,8 +379,14 @@ def new_configuration(self, specs):
                 spec.concretize()
             as_dict = {"spec": spec.to_dict(hash=ht.full_hash),
                        "spack_version": self.spack_version}
+
+            if self.save_local:
+                filename = "spec-%s-%s-config.json" % (spec.name, spec.version)
+                self.save(sjson.dump(as_dict), filename)
+            else:
                 response = self.do_request("specs/new/", data=sjson.dump(as_dict))
                 configs[spec.package.name] = response.get('data', {})
+
         return configs
 
     def new_build(self, spec):
@@ -384,6 +423,27 @@ def get_build_id(self, spec, return_response=False, spec_exists=True):
             spec_file = os.path.join(meta_dir, "spec.yaml")
             data['spec'] = syaml.load(read_file(spec_file))
 
+        if self.save_local:
+            return self.get_local_build_id(data, full_hash, return_response)
+        return self.get_server_build_id(data, full_hash, return_response)
+
+    def get_local_build_id(self, data, full_hash, return_response):
+        """
+        Generate a local build id based on hashing the expected data
+        """
+        hasher = hashlib.md5()
+        hasher.update(str(data).encode('utf-8'))
+        bid = hasher.hexdigest()
+        filename = "build-metadata-%s.json" % full_hash
+        response = self.save(sjson.dump(data), filename)
+        if return_response:
+            return response
+        return bid
+
+    def get_server_build_id(self, data, full_hash, return_response=False):
+        """
+        Retrieve a build id from the spack monitor server
+        """
         response = self.do_request("builds/new/", data=sjson.dump(data))
 
         # Add the build id to the lookup
@@ -403,6 +463,10 @@ def update_build(self, spec, status="SUCCESS"):
         successful install. This endpoint can take a general status to update.
         """
         data = {"build_id": self.get_build_id(spec), "status": status}
+        if self.save_local:
+            filename = "build-%s-status.json" % data['build_id']
+            return self.save(sjson.dump(data), filename)
+
         return self.do_request("builds/update/", data=sjson.dump(data))
 
     def fail_task(self, spec):
@@ -444,6 +508,10 @@ def send_phase(self, pkg, phase_name, phase_output_file, status):
                      "output": read_file(phase_output_file),
                      "phase_name": phase_name})
 
+        if self.save_local:
+            filename = "build-%s-phase-%s.json" % (data['build_id'], phase_name)
+            return self.save(sjson.dump(data), filename)
+
         return self.do_request("builds/phases/update/", data=sjson.dump(data))
 
     def upload_specfile(self, filename):
@@ -459,6 +527,11 @@ def upload_specfile(self, filename):
         # We load as json just to validate it
         spec = read_json(filename)
         data = {"spec": spec, "spack_verison": self.spack_version}
+
+        if self.save_local:
+            filename = "spec-%s-%s.json" % (spec.name, spec.version)
+            return self.save(sjson.dump(data), filename)
+
         return self.do_request("specs/new/", data=sjson.dump(data))
 
 

lib/spack/spack/paths.py

@@ -54,7 +54,7 @@
 user_bootstrap_path = os.path.join(user_config_path, 'bootstrap')
 user_bootstrap_store = os.path.join(user_bootstrap_path, 'store')
 reports_path = os.path.join(user_config_path, "reports")
-
+monitor_path = os.path.join(reports_path, "monitor")
 
 opt_path        = os.path.join(prefix, "opt")
 etc_path        = os.path.join(prefix, "etc")

lib/spack/spack/test/monitor.py

@@ -0,0 +1,42 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import spack.config
+import spack.spec
+from spack.main import SpackCommand
+import pytest
+import os
+
+install = SpackCommand('install')
+
+
+@pytest.fixture(scope='session')
+def test_install_monitor_save_local(install_mockery_mutable_config,
+                                    mock_fetch, tmpdir_factory):
+    """
+    Mock installing and saving monitor results to file.
+    """
+    reports_dir = tmpdir_factory.mktemp('reports')
+    spack.config.set('config:monitor_dir', str(reports_dir))
+    out = install('--monitor', '--monitor-save-local', 'dttop')
+    assert "Successfully installed dttop" in out
+
+    # The reports directory should not be empty (timestamped folders)
+    assert os.listdir(str(reports_dir))
+
+    # Get the spec name
+    spec = spack.spec.Spec("dttop")
+    spec.concretize()
+    full_hash = spec.full_hash()
+
+    # Ensure we have monitor results saved
+    for dirname in os.listdir(str(reports_dir)):
+        dated_dir = os.path.join(str(reports_dir), dirname)
+        build_metadata = "build-metadata-%s.json" % full_hash
+        assert build_metadata in os.listdir(dated_dir)
+        spec_file = "spec-dttop-%s-config.json" % spec.version
+        assert spec_file in os.listdir(dated_dir)
+
+    spack.config.set('config:monitor_dir', "~/.spack/reports/monitor")

share/spack/spack-completion.bash

@@ -358,7 +358,7 @@ _spack_add() {
 _spack_analyze() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix"
+        SPACK_COMPREPLY="-h --help --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix"
     else
         SPACK_COMPREPLY="list-analyzers run"
     fi
@@ -1063,7 +1063,7 @@ _spack_info() {
 _spack_install() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all"
+        SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all"
     else
         _all_packages
     fi