web.py: set User-Agent (#29919)

Some servers require `User-Agent` to be set, and otherwise error with
access denied. One such example is mpich.

To fix this, set `User-Agent: Spackbot/[version]` as a header.
Apparently by convention, it should include the word `bot`.

lib/spack/spack/util/web.py

@@ -24,6 +24,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp, rename
 
+import spack
 import spack.config
 import spack.error
 import spack.url
@@ -34,6 +35,9 @@
 from spack.util.compression import ALLOWED_ARCHIVE_TYPES
 from spack.util.path import convert_to_posix_path
 
+#: User-Agent used in Request objects
+SPACK_USER_AGENT = "Spackbot/{0}".format(spack.spack_version)
+
 if sys.version_info < (3, 0):
     # Python 2 had these in the HTMLParser package.
     from HTMLParser import HTMLParseError, HTMLParser  # novm
@@ -116,7 +120,7 @@ def read_from_url(url, accept_content_type=None):
     url = url_util.format(url)
     if sys.platform == "win32" and url_scheme == "file":
         url = convert_to_posix_path(url)
-    req = Request(url)
+    req = Request(url, headers={'User-Agent': SPACK_USER_AGENT})
 
     content_type = None
     is_web_url = url_scheme in ('http', 'https')