Pipelines: Improve broken specs check (#24643)

We should not fail the generate stage simply due to the presence of
a broken-spec somewhere in the DAG.  Only fail if the known broken
spec needs to be rebuilt.

lib/spack/spack/ci.py

@@ -17,28 +17,27 @@
 from six import iteritems
 from six.moves.urllib.error import HTTPError, URLError
 from six.moves.urllib.parse import urlencode
-from six.moves.urllib.request import build_opener, HTTPHandler, Request
+from six.moves.urllib.request import HTTPHandler, Request, build_opener
 
-import llnl.util.tty as tty
 import llnl.util.filesystem as fs
-
+import llnl.util.tty as tty
 import spack
 import spack.binary_distribution as bindist
 import spack.cmd
 import spack.compilers as compilers
 import spack.config as cfg
 import spack.environment as ev
-from spack.error import SpackError
 import spack.main
 import spack.mirror
 import spack.paths
 import spack.repo
-from spack.spec import Spec
 import spack.util.executable as exe
-import spack.util.spack_yaml as syaml
-import spack.util.web as web_util
 import spack.util.gpg as gpg_util
+import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
+import spack.util.web as web_util
+from spack.error import SpackError
+from spack.spec import Spec
 
 
 JOB_RETRY_CONDITIONS = [
@@ -711,14 +710,6 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
                 release_spec_dag_hash = release_spec.dag_hash()
                 release_spec_build_hash = release_spec.build_hash()
 
-                # Check if this spec is in our list of known failures.
-                if broken_specs_url:
-                    broken_spec_path = url_util.join(
-                        broken_specs_url, release_spec_full_hash)
-                    if web_util.url_exists(broken_spec_path):
-                        known_broken_specs_encountered.append('{0} ({1})'.format(
-                            release_spec, release_spec_full_hash))
-
                 runner_attribs = find_matching_config(
                     release_spec, gitlab_ci)
 
@@ -882,6 +873,15 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
                 if prune_dag and not rebuild_spec:
                     continue
 
+                # Check if this spec is in our list of known failures, now that
+                # we know this spec needs a rebuild
+                if broken_specs_url:
+                    broken_spec_path = url_util.join(
+                        broken_specs_url, release_spec_full_hash)
+                    if web_util.url_exists(broken_spec_path):
+                        known_broken_specs_encountered.append('{0} ({1})'.format(
+                            release_spec, release_spec_full_hash))
+
                 if artifacts_root:
                     job_dependencies.append({
                         'job': generate_job_name,

lib/spack/spack/test/cmd/ci.py

@@ -6,10 +6,11 @@
 import filecmp
 import json
 import os
-import pytest
-from jsonschema import validate, ValidationError
 import shutil
 
+import pytest
+from jsonschema import ValidationError, validate
+
 import spack
 import spack.ci as ci
 import spack.cmd.buildcache as buildcache
@@ -20,13 +21,13 @@
 import spack.main
 import spack.paths as spack_paths
 import spack.repo as repo
+import spack.util.gpg
+import spack.util.spack_yaml as syaml
 from spack.schema.buildcache_spec import schema as spec_yaml_schema
 from spack.schema.database_index import schema as db_idx_schema
 from spack.schema.gitlab_ci import schema as gitlab_ci_schema
-from spack.spec import Spec, CompilerSpec
+from spack.spec import CompilerSpec, Spec
 from spack.util.mock_package import MockPackageMultiRepo
-import spack.util.spack_yaml as syaml
-import spack.util.gpg
 
 
 ci_cmd = spack.main.SpackCommand('ci')
@@ -1607,7 +1608,10 @@ def test_ci_generate_read_broken_specs_url(tmpdir, mutable_mock_env_path,
     broken-specs-url: "{0}"
     mappings:
       - match:
-          - archive-files
+          - a
+          - flatten-deps
+          - b
+          - dependency-install
         runner-attributes:
           tags:
             - donotcare