zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix S3 urlparse scheme error and add test (#24952)

Browse Source
This commit is contained in:
Harmen Stoppels 2021-07-19 13:39:17 +02:00 committed by GitHub
parent 1bf9c10f0c
commit fc50e04b59
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/spack/spack/test/web.py
View File

@ -279,3 +279,8 @@ def mock_create_s3_session(url):
fake_s3_url_does_not_exist = 's3://my-bucket/subdirectory/my-notfound-file' fake_s3_url_does_not_exist = 's3://my-bucket/subdirectory/my-notfound-file'
assert(not spack.util.web.url_exists(fake_s3_url_does_not_exist)) assert(not spack.util.web.url_exists(fake_s3_url_does_not_exist))
def test_s3_url_parsing():
assert(spack.util.s3._parse_s3_endpoint_url("example.com") == 'https://example.com')
assert(spack.util.s3._parse_s3_endpoint_url("http://example.com") == 'http://example.com')

12
lib/spack/spack/util/s3.py
View File

@ -11,6 +11,13 @@
import spack.util.url as url_util import spack.util.url as url_util
def _parse_s3_endpoint_url(endpoint_url):
if not urllib_parse.urlparse(endpoint_url, scheme='').scheme:
endpoint_url = '://'.join(('https', endpoint_url))
return endpoint_url
def create_s3_session(url): def create_s3_session(url):
url = url_util.parse(url) url = url_util.parse(url)
if url.scheme != 's3': if url.scheme != 's3':
@ -30,10 +37,7 @@ def create_s3_session(url):
endpoint_url = os.environ.get('S3_ENDPOINT_URL') endpoint_url = os.environ.get('S3_ENDPOINT_URL')
if endpoint_url: if endpoint_url:
if urllib_parse.urlparse(endpoint_url, scheme=None).scheme is None: s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url)
endpoint_url = '://'.join(('https', endpoint_url))
s3_client_args['endpoint_url'] = endpoint_url
# if no access credentials provided above, then access anonymously # if no access credentials provided above, then access anonymously
if not session.get_credentials(): if not session.get_credentials():