Bug/fix credentials s3 buildcache update (#31391)

* Add connection information to buildcache update command

Ensure that the s3 connection made when attempting to update the content
of a buildcache attempts to use the extra connection information
from the mirror creation.

* Add unique help for endpoint URL argument

Fix copy/paste error for endpoint URL help which was the same as
the access token

* Re-work URL checking for S3 mirrors

Due to the fact that nested bucket URLs would never match the string used
for checking that the mirror is the same, switch the check used.

Sort all mirror URLs by length to have the most specific cases first
and see if the desired URL "starts with" the mirror URL.

* Long line style fixes

Add execptions for long lines and fix other style errors

* Use format() function to rebuild URL

Use the format command to rebuild the url instead of crafing a
formatted string out of known values

* Add early exit for URL checking

When a valid mirror is found, break from the loop

lib/spack/spack/cmd/common/arguments.py

@@ -403,4 +403,4 @@ def add_s3_connection_args(subparser, add_help):
         default=None)
     subparser.add_argument(
         '--s3-endpoint-url',
-        help="Access Token to use to connect to this S3 mirror")
+        help="Endpoint URL to use to connect to this S3 mirror")

lib/spack/spack/util/s3.py

@@ -14,9 +14,18 @@
 def get_mirror_connection(url, url_type="push"):
     connection = {}
     # Try to find a mirror for potential connection information
-    for mirror in spack.mirror.MirrorCollection().values():
-        if "%s://%s" % (url.scheme, url.netloc) == mirror.push_url:
-            connection = mirror.to_dict()[url_type]
+    # Check to see if desired file starts with any of the mirror URLs
+    rebuilt_path = url_util.format(url)
+    # Gather dict of push URLS point to the value of the whole mirror
+    mirror_dict = {x.push_url: x for x in spack.mirror.MirrorCollection().values()}  # noqa: E501
+    # Ensure most specific URLs (longest) are presented first
+    mirror_url_keys = mirror_dict.keys()
+    mirror_url_keys = sorted(mirror_url_keys, key=len, reverse=True)
+    for mURL in mirror_url_keys:
+        # See if desired URL starts with the mirror's push URL
+        if rebuilt_path.startswith(mURL):
+            connection = mirror_dict[mURL].to_dict()[url_type]
+            break
     return connection
 
 

lib/spack/spack/util/web.py

@@ -391,7 +391,7 @@ def list_url(url, recursive=False):
                 if os.path.isfile(os.path.join(local_path, subpath))]
 
     if url.scheme == 's3':
-        s3 = s3_util.create_s3_session(url)
+        s3 = s3_util.create_s3_session(url, connection=s3_util.get_mirror_connection(url))  # noqa: E501
         if recursive:
             return list(_iter_s3_prefix(s3, url))