spack reproduce-build: accept URLs from web interface (#42261)

Sometimes the logs are too long and the copy & paste command is not
shown. In that case I'd like to just copy the failing GitLab job URL in
my browser to `spack reproduce-build <url>`.

lib/spack/spack/cmd/ci.py

@@ -6,6 +6,7 @@
 import json
 import os
 import shutil
+from urllib.parse import urlparse, urlunparse
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -157,7 +158,9 @@ def setup_parser(subparser):
         description=deindent(ci_reproduce.__doc__),
         help=spack.cmd.first_line(ci_reproduce.__doc__),
     )
-    reproduce.add_argument("job_url", help="URL of job artifacts bundle")
+    reproduce.add_argument(
+        "job_url", help="URL of GitLab job web page or artifact", type=_gitlab_artifacts_url
+    )
     reproduce.add_argument(
         "--runtime",
         help="Container runtime to use.",
@@ -792,11 +795,6 @@ def ci_reproduce(args):
     artifacts of the provided gitlab pipeline rebuild job's URL will be used to derive
     instructions for reproducing the build locally
     """
-    job_url = args.job_url
-    work_dir = args.working_dir
-    autostart = args.autostart
-    runtime = args.runtime
-
     # Allow passing GPG key for reprocuding protected CI jobs
     if args.gpg_file:
         gpg_key_url = url_util.path_to_file_url(args.gpg_file)
@@ -805,7 +803,47 @@ def ci_reproduce(args):
     else:
         gpg_key_url = None
 
-    return spack_ci.reproduce_ci_job(job_url, work_dir, autostart, gpg_key_url, runtime)
+    return spack_ci.reproduce_ci_job(
+        args.job_url, args.working_dir, args.autostart, gpg_key_url, args.runtime
+    )
+
+
+def _gitlab_artifacts_url(url: str) -> str:
+    """Take a URL either to the URL of the job in the GitLab UI, or to the artifacts zip file,
+    and output the URL to the artifacts zip file."""
+    parsed = urlparse(url)
+
+    if not parsed.scheme or not parsed.netloc:
+        raise ValueError(url)
+
+    parts = parsed.path.split("/")
+
+    if len(parts) < 2:
+        raise ValueError(url)
+
+    # Just use API endpoints verbatim, they're probably generated by Spack.
+    if parts[1] == "api":
+        return url
+
+    # If it's a URL to the job in the Gitlab UI, we may need to append the artifacts path.
+    minus_idx = parts.index("-")
+
+    # Remove repeated slashes in the remainder
+    rest = [p for p in parts[minus_idx + 1 :] if p]
+
+    # Now the format is jobs/X or jobs/X/artifacts/download
+    if len(rest) < 2 or rest[0] != "jobs":
+        raise ValueError(url)
+
+    if len(rest) == 2:
+        # replace jobs/X with jobs/X/artifacts/download
+        rest.extend(("artifacts", "download"))
+
+    # Replace the parts and unparse.
+    parts[minus_idx + 1 :] = rest
+
+    # Don't allow fragments / queries
+    return urlunparse(parsed._replace(path="/".join(parts), fragment="", query=""))
 
 
 def ci(parser, args):

lib/spack/spack/test/cmd/ci.py

@@ -16,6 +16,7 @@
 import spack
 import spack.binary_distribution
 import spack.ci as ci
+import spack.cmd.ci
 import spack.config
 import spack.environment as ev
 import spack.hash_types as ht
@@ -2028,6 +2029,43 @@ def fake_download_and_extract_artifacts(url, work_dir):
     assert expect_out in rep_out
 
 
+@pytest.mark.parametrize(
+    "url_in,url_out",
+    [
+        (
+            "https://example.com/api/v4/projects/1/jobs/2/artifacts",
+            "https://example.com/api/v4/projects/1/jobs/2/artifacts",
+        ),
+        (
+            "https://example.com/spack/spack/-/jobs/123456/artifacts/download",
+            "https://example.com/spack/spack/-/jobs/123456/artifacts/download",
+        ),
+        (
+            "https://example.com/spack/spack/-/jobs/123456",
+            "https://example.com/spack/spack/-/jobs/123456/artifacts/download",
+        ),
+        (
+            "https://example.com/spack/spack/-/jobs/////123456////?x=y#z",
+            "https://example.com/spack/spack/-/jobs/123456/artifacts/download",
+        ),
+    ],
+)
+def test_reproduce_build_url_validation(url_in, url_out):
+    assert spack.cmd.ci._gitlab_artifacts_url(url_in) == url_out
+
+
+def test_reproduce_build_url_validation_fails():
+    """Wrong URLs should cause an exception"""
+    with pytest.raises(SystemExit):
+        ci_cmd("reproduce-build", "example.com/spack/spack/-/jobs/123456/artifacts/download")
+
+    with pytest.raises(SystemExit):
+        ci_cmd("reproduce-build", "https://example.com/spack/spack/-/issues")
+
+    with pytest.raises(SystemExit):
+        ci_cmd("reproduce-build", "https://example.com/spack/spack/-")
+
+
 @pytest.mark.parametrize(
     "subcmd", [(""), ("generate"), ("rebuild-index"), ("rebuild"), ("reproduce-build")]
 )