wip: working for installer tests

Signed-off-by: Gregory Becker <becker33@llnl.gov>

lib/spack/spack/ci/__init__.py

@@ -215,6 +215,7 @@ def create_external_pruner() -> Callable[[spack.spec.Spec], RebuildDecision]:
     """Return a filter that prunes external specs"""
 
     def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
+        print(s.name, "external:", s.external)
         if not s.external:
             return RebuildDecision(True, "not external")
         return RebuildDecision(False, "external spec")

lib/spack/spack/cmd/install.py

@@ -16,6 +16,8 @@
 import spack.concretize
 import spack.config
 import spack.environment as ev
+import spack.hooks
+import spack.hooks.report
 import spack.paths
 import spack.report
 import spack.spec
@@ -329,13 +331,10 @@ def install(parser, args):
 
     arguments.sanitize_reporter_options(args)
 
-    def reporter_factory(specs):
-        if args.log_format is None:
-            return lang.nullcontext()
-
-        return spack.report.build_context_manager(
-            reporter=args.reporter(), filename=report_filename(args, specs=specs), specs=specs
-        )
+    # TODO: This is hacky as hell
+    if args.log_format is not None:
+        spack.hooks.report.reporter = args.reporter()
+        spack.hooks.report.report_file = args.log_file
 
     install_kwargs = install_kwargs_from_args(args)
 
@@ -346,9 +345,9 @@ def reporter_factory(specs):
 
     try:
         if env:
-            install_with_active_env(env, args, install_kwargs, reporter_factory)
+            install_with_active_env(env, args, install_kwargs)
         else:
-            install_without_active_env(args, install_kwargs, reporter_factory)
+            install_without_active_env(args, install_kwargs)
     except InstallError as e:
         if args.show_log_on_error:
             _dump_log_on_error(e)
@@ -382,7 +381,7 @@ def _maybe_add_and_concretize(args, env, specs):
         env.write(regenerate=False)
 
 
-def install_with_active_env(env: ev.Environment, args, install_kwargs, reporter_factory):
+def install_with_active_env(env: ev.Environment, args, install_kwargs):
     specs = spack.cmd.parse_specs(args.spec)
 
     # The following two commands are equivalent:
@@ -416,7 +415,6 @@ def install_with_active_env(env: ev.Environment, args, install_kwargs, reporter_
         install_kwargs["overwrite"] = [spec.dag_hash() for spec in specs_to_install]
 
     try:
-        with reporter_factory(specs_to_install):
         env.install_specs(specs_to_install, **install_kwargs)
     finally:
         if env.views:
@@ -461,13 +459,12 @@ def concrete_specs_from_file(args):
     return result
 
 
-def install_without_active_env(args, install_kwargs, reporter_factory):
+def install_without_active_env(args, install_kwargs):
     concrete_specs = concrete_specs_from_cli(args, install_kwargs) + concrete_specs_from_file(args)
 
     if len(concrete_specs) == 0:
         tty.die("The `spack install` command requires a spec to install.")
 
-    with reporter_factory(concrete_specs):
     if args.overwrite:
         require_user_confirmation_for_overwrite(concrete_specs, args)
         install_kwargs["overwrite"] = [spec.dag_hash() for spec in concrete_specs]

lib/spack/spack/hooks/__init__.py

@@ -27,6 +27,7 @@
 class _HookRunner:
     #: Order in which hooks are executed
     HOOK_ORDER = [
+        "spack.hooks.report",
         "spack.hooks.module_file_generation",
         "spack.hooks.licensing",
         "spack.hooks.sbang",
@@ -67,3 +68,6 @@ def __call__(self, *args, **kwargs):
 
 pre_uninstall = _HookRunner("pre_uninstall")
 post_uninstall = _HookRunner("post_uninstall")
+
+pre_installer = _HookRunner("pre_installer")
+post_installer = _HookRunner("post_installer")

lib/spack/spack/hooks/report.py

@@ -0,0 +1,263 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Hooks to produce reports of spec installations"""
+import collections
+import gzip
+import os
+import time
+import traceback
+
+import llnl.util.filesystem as fs
+
+import spack.build_environment
+import spack.util.spack_json as sjson
+
+reporter = None
+report_file = None
+
+Property = collections.namedtuple("Property", ["name", "value"])
+
+
+class Record(dict):
+    def __getattr__(self, name):
+        # only called if no attribute exists
+        if name in self:
+            return self[name]
+        raise AttributeError(f"RequestRecord for {self.name} has no attribute {name}")
+
+    def __setattr__(self, name, value):
+        if name.startswith("_"):
+            super().__setattr__(name, value)
+        else:
+            self[name] = value
+
+
+class RequestRecord(Record):
+    def __init__(self, spec):
+        super().__init__()
+        self.name = spec.name
+        self.errors = None
+        self.nfailures = None
+        self.npackages = None
+        self.time = None
+        self.timestamp = time.strftime("%a, d %b %Y %H:%M:%S", time.gmtime())
+        self.properties = [
+            Property("architecture", spec.architecture),
+            Property("compiler", spec.compiler),
+        ]
+        self.packages = []
+        self._seen = set()
+
+    def append_record(self, record, key):
+        self.packages.append(record)
+        self._seen.add(key)
+
+    def seen(self, key):
+        return key in self._seen
+
+    def summarize(self):
+        self.npackages = len(self.packages)
+        self.nfailures = len([r for r in self.packages if r.result == "failure"])
+        self.nerrors = len([r for r in self.packages if r.result == "error"])
+        self.time = sum(float(r.elapsed_time or 0.0) for r in self.packages)
+
+
+class SpecRecord(Record):
+    pass
+
+
+class InstallRecord(SpecRecord):
+    def __init__(self, spec):
+        super().__init__()
+        self._spec = spec
+        self._package = spec.package
+        self._start_time = time.time()
+        self.name = spec.name
+        self.id = spec.dag_hash()
+        self.elapsed_time = None
+        self.result = None
+        self.message = None
+        self.installed_from_binary_cache = None
+
+    def fetch_log(self):
+        try:
+            if os.path.exists(self._package.install_log_path):
+                stream = gzip.open(self._package.install_log_path, "rt", encoding="utf-8")
+            else:
+                stream = open(self._package.log_path, encoding="utf-8")
+            with stream as f:
+                return f.read()
+        except OSError:
+            return f"Cannot open log for {self._spec.cshort_spec}"
+
+    def fetch_time(self):
+        try:
+            with open(self._package.times_log_path, "r", encoding="utf-8") as f:
+                data = sjson.load(f.read())
+            return data["total"]
+        except Exception:
+            return None
+
+    def skip(self, msg):
+        self.result = "skipped"
+        self.elapsed_time = 0.0
+        self.message = msg
+
+    def succeed(self):
+        self.result = "success"
+        self.stdout = self.fetch_log()
+        self.installed_from_binary_cache = self._package.installed_from_binary_cache
+        self.elapsed_time = self.fetch_time()
+
+    def fail(self, exc):
+        if isinstance(exc, spack.build_environment.InstallError):
+            self.result = "failure"
+            self.message = exc.message or "Installation failure"
+            self.exception = exc.traceback
+        else:
+            self.result = "error"
+            self.message = str(exc) or "Unknown error"
+            self.exception = traceback.format_exc()
+        self.stdout = self.fetch_log() + self.message
+
+
+requests = {}
+
+
+def pre_installer(specs):
+    global requests
+
+    for root in specs:
+        request = RequestRecord(root)
+        requests[root.dag_hash()] = request
+
+        for dep in filter(lambda x: x.installed, root.traverse()):
+            record = InstallRecord(dep)
+            record.skip(msg="Spec already installed")
+            request.append_record(record, dep.dag_hash())
+
+
+def post_installer(specs, hashes_to_failures):
+    global requests
+    global report_file
+    global reporter
+
+    try:
+        for root in specs:
+            request = requests[root.dag_hash()]
+
+            # Associate all dependency jobs with this request
+            for dep in root.traverse():
+                if request.seen(dep.dag_hash()):
+                    continue  # Already handled
+
+                record = InstallRecord(dep)
+                if dep.dag_hash() in hashes_to_failures:
+                    record.fail(hashes_to_failures[dep.dag_hash()])
+                elif dep.installed:
+                    record.succeed()
+                else:
+                    # This package was never reached because of an earlier failure
+                    continue
+                request.append_record(record, dep.dag_hash())
+
+            # Aggregate request-level data
+            request.summarize()
+
+        # Write the actual report
+        if not report_file:
+            basename = specs[0].format("test-{name}-{version}-{hash}.xml")
+            dirname = os.path.join(spack.paths.reports_path, "junit")
+            fs.mkdirp(dirname)
+            report_file = os.path.join(dirname, basename)
+        if reporter:
+            reporter.build_report(report_file, specs=list(requests.values()))
+
+    finally:
+        # Clean up after ourselves
+        requests = {}
+        reporter = None
+        report_file = None
+
+
+# This is not thread safe, but that should be ok
+# We only have one top-level thread launching build requests, and all parallelism
+# is between the jobs of different requests
+# requests: Dict[str, RequestRecord] = {}
+# specs: Dict[str, InstallRecord] = {}
+
+
+# def pre_installer(specs):
+#     global requests
+#     global specs
+
+#     for spec in specs:
+#         record = RequestRecord(spec)
+#         requests[spec.dag_hash()] = record
+
+#         for dep in filter(lambda x: x.installed, spec.traverse()):
+#             spec_record = InstallRecord(dep)
+#             spec_record.elapsed_time = "0.0"
+#             spec_record.result = "skipped"
+#             spec_record.message = "Spec already installed"
+#             specs[dep.dag_hash()] = spec_record
+
+# def pre_install(spec):
+#     global specs
+
+#     specs[spec.dag_hash()] = InstallRecord(spec)
+
+
+# def post_install(spec, explicit: bool):
+#     global specs
+
+#     record = specs[spec.dag_hash()]
+#     record.result = "success"
+#     record.stdout = record.fetch_log()
+#     record.installed_from_binary_cache = record._package.installed_from_binary_cache
+#     record.elapsed_time = time.time() - record._start_time
+
+
+# def post_failure(spec, error):
+#     global specs
+
+#     record = specs[spec.dag_hash()]
+#     if isinstance(error, spack.build_environment.InstallError):
+#         record.result = "failure"
+#         record.message = exc.message or "Installation failure"
+#         record.exception = exc.traceback
+#     else:
+#         record.result = "error"
+#         record.message = str(exc) or "Unknown error"
+#         record.exception = traceback.format_exc()
+#     record.stdout = record.fetch_log() + record.message
+#     record.elapsed_time = time.time() - record._start_time
+
+
+# def post_installer(specs):
+#     global requests
+#     global specs
+#     global reporter
+#     global report_file
+
+#     for spec in specs:
+#         # Find all associated spec records
+#         request_record = requests[spec.dag_hash()]
+#         for dep in spec.traverse(root=True):
+#             spec_record = specs[dep.dag_hash()]
+#             request_record.records.append(spec_record)
+
+#         # Aggregate statistics
+#         request_record.npackages = len(request_record.records)
+#         request_record.nfailures = len([r for r in request_record.records if r.result == "failure"])
+#         request_record.errors = len([r for r in request_record.records if r.result == "error"])
+#         request_record.time = sum(float(r.elapsed_time) for r in request_record.records)
+
+#     # Write the actual report
+#     filename = report_file or specs[0].name
+#     reporter.build_report(filename, specs=specs)
+
+#     # Clean up after ourselves
+#     requests = {}
+#     specs = {}

lib/spack/spack/installer.py

@@ -2014,11 +2014,13 @@ def _install_action(self, task: Task) -> InstallAction:
 
     def install(self) -> None:
         """Install the requested package(s) and or associated dependencies."""
+        spack.hooks.pre_installer([r.pkg.spec for r in self.build_requests])
 
         self._init_queue()
         fail_fast_err = "Terminating after first install failure"
         single_requested_spec = len(self.build_requests) == 1
         failed_build_requests = []
+        failed_tasks = []  # self.failed tracks dependents of failed tasks, here only failures
 
         install_status = InstallStatus(len(self.build_pq))
 
@@ -2171,13 +2173,24 @@ def install(self) -> None:
             except KeyboardInterrupt as exc:
                 # The build has been terminated with a Ctrl-C so terminate
                 # regardless of the number of remaining specs.
+                failed_tasks.append((pkg, exc))
                 tty.error(
                     f"Failed to install {pkg.name} due to " f"{exc.__class__.__name__}: {str(exc)}"
                 )
+                hashes_to_failures = {pkg.spec.dag_hash(): exc for pkg, exc in failed_tasks}
+                spack.hooks.post_installer(
+                    [r.pkg.spec for r in self.build_requests], hashes_to_failures
+                )
+                print("DDDDDD")
                 raise
 
             except binary_distribution.NoChecksumException as exc:
                 if task.cache_only:
+                    failed_tasks.append((pkg, exc))
+                    hashes_to_failures = {pkg.spec.dag_hash(): exc for pkg, exc in failed_tasks}
+                    spack.hooks.post_installer(
+                        [r.pkg.spec for r in self.build_requests], hashes_to_failures
+                    )
                     raise
 
                 # Checking hash on downloaded binary failed.
@@ -2192,6 +2205,7 @@ def install(self) -> None:
 
             except (Exception, SystemExit) as exc:
                 self._update_failed(task, True, exc)
+                failed_tasks.append((pkg, exc))
 
                 # Best effort installs suppress the exception and mark the
                 # package as a failure.
@@ -2204,8 +2218,14 @@ def install(self) -> None:
                         f"Failed to install {pkg.name} due to "
                         f"{exc.__class__.__name__}: {str(exc)}"
                     )
+
                 # Terminate if requested to do so on the first failure.
                 if self.fail_fast:
+                    hashes_to_failures = {pkg.spec.dag_hash(): exc for pkg, exc in failed_tasks}
+                    spack.hooks.post_installer(
+                        [r.pkg.spec for r in self.build_requests], hashes_to_failures
+                    )
+                    print("AAAAAAA")
                     raise spack.error.InstallError(
                         f"{fail_fast_err}: {str(exc)}", pkg=pkg
                     ) from exc
@@ -2213,8 +2233,15 @@ def install(self) -> None:
                 # Terminate when a single build request has failed, or summarize errors later.
                 if task.is_build_request:
                     if single_requested_spec:
+                        hashes_to_failures = {
+                            pkg.spec.dag_hash(): exc for pkg, exc in failed_tasks
+                        }
+                        spack.hooks.post_installer(
+                            [r.pkg.spec for r in self.build_requests], hashes_to_failures
+                        )
+                        print("BBBBB")
                         raise
-                    failed_build_requests.append((pkg, pkg_id, str(exc)))
+                    failed_build_requests.append((pkg, pkg_id, exc))
 
             finally:
                 # Remove the install prefix if anything went wrong during
@@ -2238,9 +2265,13 @@ def install(self) -> None:
             if request.install_args.get("install_package") and request.pkg_id not in self.installed
         ]
 
+        hashes_to_failures = {pkg.spec.dag_hash(): exc for pkg, exc in failed_tasks}
+        spack.hooks.post_installer([r.pkg.spec for r in self.build_requests], hashes_to_failures)
+        print("CCCCC", failed_build_requests)
+
         if failed_build_requests or missing:
             for _, pkg_id, err in failed_build_requests:
-                tty.error(f"{pkg_id}: {err}")
+                tty.error(f"{pkg_id}: {str(err)}")
 
             for _, pkg_id in missing:
                 tty.error(f"{pkg_id}: Package was not installed")

lib/spack/spack/reporters/cdash.py

@@ -217,6 +217,7 @@ def build_report_for_package(self, report_dir, package, duration):
             nerrors = len(errors)
 
             if nerrors > 0:
+                print("NERRORS")
                 self.success = False
                 if phase == "configure":
                     report_data[phase]["status"] = 1
@@ -410,6 +411,7 @@ def concretization_report(self, report_dir, msg):
         self.current_package_name = self.base_buildname
         self.upload(output_filename)
         self.success = False
+        print("CONCRETIZATION")
         self.finalize_report()
 
     def initialize_report(self, report_dir):

lib/spack/spack/test/cmd/install.py

@@ -450,6 +450,8 @@ def just_throw(*args, **kwargs):
 
     content = filename.open().read()
 
+    print(content)
+
     # Only libelf error is reported (through libdwarf root spec). libdwarf
     # install is skipped and it is not an error.
     assert 'tests="1"' in content