diff --git a/setup.py b/setup.py index 4597a95..c794859 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ setup( 'pluggy>0.7<1.0', 'passlib', 'backoff', + 'requests', 'jupyterhub-traefik-proxy==0.1.*' ], entry_points={ diff --git a/tljh/conda.py b/tljh/conda.py index 2a30a2b..35d45fc 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -7,7 +7,7 @@ import json import hashlib import contextlib import tempfile -import urllib.request +import requests from distutils.version import LooseVersion as V @@ -50,7 +50,8 @@ def download_miniconda_installer(version, md5sum): """ with tempfile.NamedTemporaryFile() as f: installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) - urllib.request.urlretrieve(installer_url, f.name) + with open(f.name, 'wb') as f: + f.write(requests.get(installer_url).content) if md5_file(f.name) != md5sum: raise Exception('md5 hash mismatch! Downloaded file corrupted') diff --git a/tljh/installer.py b/tljh/installer.py index 91395cf..0e6dee7 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -8,10 +8,9 @@ import secrets import subprocess import sys import time -from urllib.error import HTTPError -from urllib.request import urlopen, URLError import pluggy +import requests from tljh import ( apt, @@ -293,20 +292,20 @@ def ensure_jupyterhub_running(times=20): for i in range(times): try: logger.info('Waiting for JupyterHub to come up ({}/{} tries)'.format(i + 1, times)) - urlopen('http://127.0.0.1') + requests.get('http://127.0.0.1') return - except HTTPError as h: - if h.code in [404, 502, 503]: + except requests.HTTPError as h: + if h.response.status_code in [404, 502, 503]: # May be transient time.sleep(1) continue # Everything else should immediately abort raise - except URLError as e: - if isinstance(e.reason, ConnectionRefusedError): + except requests.ConnectionError: # Hub isn't up yet, sleep & loop time.sleep(1) continue + except Exception: # Everything else should immediately abort raise diff --git a/tljh/traefik.py b/tljh/traefik.py index 4e05b7a..e7bd78d 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -1,11 +1,11 @@ """Traefik installation and setup""" import hashlib import os -from urllib.request import urlretrieve, ContentTooShortError from jinja2 import Template from passlib.apache import HtpasswdFile import backoff +import requests from tljh.configurer import load_config @@ -27,11 +27,15 @@ def checksum_file(path): hasher.update(chunk) return hasher.hexdigest() +def fatal_error(e): + # Retry only when connection is reset or we think we didn't download entire file + return str(e) != "ContentTooShort" and not isinstance(e, ConnectionResetError) + @backoff.on_exception( backoff.expo, - # Retry when connection is reset or we think we didn't download entire file - (ConnectionResetError, ContentTooShortError), - max_tries=2 + Exception, + max_tries=2, + giveup=fatal_error ) def ensure_traefik_binary(prefix): """Download and install the traefik binary""" @@ -53,7 +57,11 @@ def ensure_traefik_binary(prefix): ) print(f"Downloading traefik {traefik_version}...") # download the file - urlretrieve(traefik_url, traefik_bin) + response = requests.get(traefik_url) + if response.status_code == 206: + raise Exception("ContentTooShort") + with open(traefik_bin, 'wb') as f: + f.write(response.content) os.chmod(traefik_bin, 0o755) # verify that we got what we expected