Picklable HTTPError (#39285)

lib/spack/spack/test/web.py

@@ -3,7 +3,10 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections
+import email.message
 import os
+import pickle
+import urllib.request
 
 import pytest
 
@@ -339,3 +342,25 @@ def get_s3_session(url, method="fetch"):
 def test_s3_url_parsing():
     assert spack.util.s3._parse_s3_endpoint_url("example.com") == "https://example.com"
     assert spack.util.s3._parse_s3_endpoint_url("http://example.com") == "http://example.com"
+
+
+def test_detailed_http_error_pickle(tmpdir):
+    tmpdir.join("response").write("response")
+
+    headers = email.message.Message()
+    headers.add_header("Content-Type", "text/plain")
+
+    # Use a temporary file object as a response body
+    with open(str(tmpdir.join("response")), "rb") as f:
+        error = spack.util.web.DetailedHTTPError(
+            urllib.request.Request("http://example.com"), 404, "Not Found", headers, f
+        )
+
+        deserialized = pickle.loads(pickle.dumps(error))
+
+    assert isinstance(deserialized, spack.util.web.DetailedHTTPError)
+    assert deserialized.code == 404
+    assert deserialized.filename == "http://example.com"
+    assert deserialized.reason == "Not Found"
+    assert str(deserialized.info()) == str(headers)
+    assert str(deserialized) == str(error)

lib/spack/spack/util/web.py

@@ -17,6 +17,7 @@
 import urllib.parse
 from html.parser import HTMLParser
 from pathlib import Path, PurePosixPath
+from typing import IO, Optional
 from urllib.error import HTTPError, URLError
 from urllib.request import HTTPSHandler, Request, build_opener
 
@@ -40,7 +41,9 @@
 
 
 class DetailedHTTPError(HTTPError):
-    def __init__(self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp) -> None:
+    def __init__(
+        self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp: Optional[IO]
+    ) -> None:
         self.req = req
         super().__init__(req.get_full_url(), code, msg, hdrs, fp)
 
@@ -48,7 +51,13 @@ def __str__(self):
         # Note: HTTPError, is actually a kind of non-seekable response object, so
         # best not to read the response body here (even if it may include a human-readable
         # error message).
-        return f"{self.req.get_method()} {self.url} returned {self.code}: {self.msg}"
+        # Note: use self.filename, not self.url, because the latter requires fp to be an
+        # IO object, which is not the case after unpickling.
+        return f"{self.req.get_method()} {self.filename} returned {self.code}: {self.msg}"
+
+    def __reduce__(self):
+        # fp is an IO object and not picklable, the rest should be.
+        return DetailedHTTPError, (self.req, self.code, self.msg, self.hdrs, None)
 
 
 class SpackHTTPDefaultErrorHandler(urllib.request.HTTPDefaultErrorHandler):