From e353ab80c3847ccf4f0feed4585dbbbe3eba65ac Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 21 Mar 2023 16:23:41 +0100 Subject: [PATCH 01/12] traefik 2.9.9 - traefik releases are tarballs now - move traefik platform check to download function for easier unit testing on unsupported platforms --- tljh/traefik.py | 84 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/tljh/traefik.py b/tljh/traefik.py index c93a9fa..e5aae6e 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -1,7 +1,11 @@ """Traefik installation and setup""" import hashlib +import io +import logging import os +import tarfile from glob import glob +from subprocess import run import backoff import requests @@ -13,28 +17,33 @@ from tljh.configurer import _merge_dictionaries, load_config from .config import CONFIG_DIR -# traefik 2.7.x is not supported yet, use v1.7.x for now -# see: https://github.com/jupyterhub/traefik-proxy/issues/97 +logger = logging.getLogger("tljh") + machine = os.uname().machine if machine == "aarch64": - plat = "linux-arm64" + plat = "linux_arm64" elif machine == "x86_64": - plat = "linux-amd64" + plat = "linux_amd64" else: - raise OSError(f"Error. Platform: {os.uname().sysname} / {machine} Not supported.") -traefik_version = "1.7.33" + plat = None + +traefik_version = "2.9.9" # record sha256 hashes for supported platforms here checksums = { - "linux-amd64": "314ffeaa4cd8ed6ab7b779e9b6773987819f79b23c28d7ab60ace4d3683c5935", - "linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1", + "linux_amd64": "141db1434ae76890915486a4bc5ecf3dbafc8ece78984ce1a8db07737c42db88", + "linux_arm64": "0a65ead411307669916ba629fa13f698acda0b2c5387abe0309b43e168e4e57f", } -def checksum_file(path): +def checksum_file(path_or_file): """Compute the sha256 checksum of a path""" hasher = hashlib.sha256() - with open(path, "rb") as f: + if hasattr(path_or_file, "read"): + f = path_or_file + else: + f = open(path_or_file, "rb") + with f: for chunk in iter(lambda: f.read(4096), b""): hasher.update(chunk) return hasher.hexdigest() @@ -45,39 +54,68 @@ def fatal_error(e): return str(e) != "ContentTooShort" and not isinstance(e, ConnectionResetError) +def check_traefik_version(traefik_bin): + """Check the traefik version from `traefik version` output""" + + try: + version_out = run([traefik_bin, "version"], capture=True, text=True) + except (FileNotFoundError, OSError) as e: + logger.debug(f"Failed to get traefik version: {e}") + return False + versions = {} + for line in version_out.splitlines(): + before, _, after = line.partition(":") + key = before.strip() + if key.lower() == "version": + version = after.strip() + if version == traefik_version: + logger.debug(f"Found {traefik_bin} {version}") + return True + else: + logger.info( + f"Found {traefik_bin} version {version} != {traefik_version}" + ) + return False + + logger.debug(f"Failed to extract traefik version from: {version_out}") + return False + + @backoff.on_exception(backoff.expo, Exception, max_tries=2, giveup=fatal_error) def ensure_traefik_binary(prefix): """Download and install the traefik binary to a location identified by a prefix path such as '/opt/tljh/hub/'""" + if plat is None: + raise OSError( + f"Error. Platform: {os.uname().sysname} / {machine} Not supported." + ) + traefik_bin_dir = os.path.join(prefix, "bin") traefik_bin = os.path.join(prefix, "bin", "traefik") if os.path.exists(traefik_bin): - checksum = checksum_file(traefik_bin) - if checksum == checksums[plat]: - # already have the right binary - # ensure permissions and we're done - os.chmod(traefik_bin, 0o755) + if check_traefik_version(traefik_bin): return else: - print(f"checksum mismatch on {traefik_bin}") os.remove(traefik_bin) traefik_url = ( "https://github.com/containous/traefik/releases" - f"/download/v{traefik_version}/traefik_{plat}" + f"/download/v{traefik_version}/traefik_v{traefik_version}_{plat}.tar.gz" ) - print(f"Downloading traefik {traefik_version}...") + logger.info(f"Downloading traefik {traefik_version} from {traefik_url}...") # download the file response = requests.get(traefik_url) + response.raise_for_status() if response.status_code == 206: raise Exception("ContentTooShort") - with open(traefik_bin, "wb") as f: - f.write(response.content) - os.chmod(traefik_bin, 0o755) # verify that we got what we expected - checksum = checksum_file(traefik_bin) + checksum = checksum_file(io.BytesIO(response.content)) if checksum != checksums[plat]: - raise OSError(f"Checksum failed {traefik_bin}: {checksum} != {checksums[plat]}") + raise OSError(f"Checksum failed {traefik_url}: {checksum} != {checksums[plat]}") + + with tarfile.open(fileobj=io.BytesIO(response.content)) as tf: + tf.extract("traefik", path=traefik_bin_dir) + os.chmod(traefik_bin, 0o755) def compute_basic_auth(username, password): From a58956f14b9361c020d94567a6fb42c6127071db Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 15 May 2023 10:53:53 +0200 Subject: [PATCH 02/12] update for traefik v2, treafik-proxy v1 - tls config is no longer allowed in static config file, add separate dynamic config - no longer need to persist auth config ourselves (TraefikProxy handles this) - make sure to reload proxy before reloading hub in tests --- integration-tests/conftest.py | 2 +- integration-tests/test_proxy.py | 63 +++++++---- setup.py | 3 +- tests/test_configurer.py | 10 +- tests/test_traefik.py | 194 +++++++++++++++++++++----------- tljh/config.py | 5 +- tljh/configurer.py | 9 +- tljh/jupyterhub_config.py | 9 +- tljh/traefik-dynamic.toml.tpl | 25 ++++ tljh/traefik.py | 47 ++++---- tljh/traefik.toml.tpl | 83 ++++++-------- 11 files changed, 272 insertions(+), 178 deletions(-) create mode 100644 tljh/traefik-dynamic.toml.tpl diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 123be8d..b48e032 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -25,5 +25,5 @@ def preserve_config(request): f.write(save_config) elif os.path.exists(CONFIG_FILE): os.remove(CONFIG_FILE) - reload_component("hub") reload_component("proxy") + reload_component("hub") diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index 6aca51e..cc9d766 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -19,9 +19,7 @@ from tljh.config import ( def send_request(url, max_sleep, validate_cert=True, username=None, password=None): - resp = None for i in range(max_sleep): - time.sleep(i) try: req = HTTPRequest( url, @@ -32,12 +30,12 @@ def send_request(url, max_sleep, validate_cert=True, username=None, password=Non follow_redirects=True, max_redirects=15, ) - resp = HTTPClient().fetch(req) - break + return HTTPClient().fetch(req) except Exception as e: + if i + 1 == max_sleep: + raise print(e) - - return resp + time.sleep(i) def test_manual_https(preserve_config): @@ -104,37 +102,51 @@ def test_extra_traefik_config(): os.makedirs(dynamic_config_dir, exist_ok=True) extra_static_config = { - "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}}, - "api": {"dashboard": True, "entrypoint": "no_auth_api"}, + "entryPoints": {"alsoHub": {"address": "127.0.0.1:9999"}}, } extra_dynamic_config = { - "frontends": { - "test": { - "backend": "test", - "routes": { - "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"} + "http": { + "middlewares": { + "testHubStripPrefix": { + "stripPrefix": {"prefixes": ["/the/hub/runs/here/too"]} + } + }, + "routers": { + "test1": { + "rule": "PathPrefix(`/hub`)", + "entryPoints": ["alsoHub"], + "service": "test", }, - } - }, - "backends": { - # redirect to hub - "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}} + "test2": { + "rule": "PathPrefix(`/the/hub/runs/here/too`)", + "middlewares": ["testHubStripPrefix"], + "entryPoints": ["http"], + "service": "test", + }, + }, + "services": { + "test": { + "loadBalancer": { + # forward requests to the hub + "servers": [{"url": "http://127.0.0.1:15001"}] + } + } + }, }, } success = False for i in range(5): - time.sleep(i) try: with pytest.raises(HTTPClientError, match="HTTP 401: Unauthorized"): - # The default dashboard entrypoint requires authentication, so it should fail - req = HTTPRequest("http://127.0.0.1:8099/dashboard/", method="GET") - HTTPClient().fetch(req) + # The default api entrypoint requires authentication, so it should fail + HTTPClient().fetch("http://localhost:8099/api") success = True break - except Exception: - pass + except Exception as e: + print(e) + time.sleep(i) assert success == True @@ -153,8 +165,9 @@ def test_extra_traefik_config(): # load the extra config reload_component("proxy") + # check hub page # the new dashboard entrypoint shouldn't require authentication anymore - resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5) + resp = send_request(url="http://127.0.0.1:9999/hub/login", max_sleep=5) assert resp.code == 200 # test extra dynamic config diff --git a/setup.py b/setup.py index 12d7240..255ab75 100644 --- a/setup.py +++ b/setup.py @@ -14,11 +14,10 @@ setup( "ruamel.yaml==0.17.*", "jinja2", "pluggy==1.*", - "passlib", "backoff", "requests", "bcrypt", - "jupyterhub-traefik-proxy==0.3.*", + "jupyterhub-traefik-proxy==1.0.0b3", ], entry_points={ "console_scripts": [ diff --git a/tests/test_configurer.py b/tests/test_configurer.py index c348236..29c073a 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -156,8 +156,8 @@ def test_traefik_api_default(): """ c = apply_mock_config({}) - assert c.TraefikTomlProxy.traefik_api_username == "api_admin" - assert len(c.TraefikTomlProxy.traefik_api_password) == 0 + assert c.TraefikProxy.traefik_api_username == "api_admin" + assert len(c.TraefikProxy.traefik_api_password) == 0 def test_set_traefik_api(): @@ -167,8 +167,8 @@ def test_set_traefik_api(): c = apply_mock_config( {"traefik_api": {"username": "some_user", "password": "1234"}} ) - assert c.TraefikTomlProxy.traefik_api_username == "some_user" - assert c.TraefikTomlProxy.traefik_api_password == "1234" + assert c.TraefikProxy.traefik_api_username == "some_user" + assert c.TraefikProxy.traefik_api_password == "1234" def test_cull_service_default(): @@ -268,7 +268,7 @@ def test_load_secrets(tljh_dir): tljh_config = configurer.load_config() assert tljh_config["traefik_api"]["password"] == "traefik-password" c = apply_mock_config(tljh_config) - assert c.TraefikTomlProxy.traefik_api_password == "traefik-password" + assert c.TraefikProxy.traefik_api_password == "traefik-password" def test_auth_native(): diff --git a/tests/test_traefik.py b/tests/test_traefik.py index 0cb406b..f78898f 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -15,30 +15,51 @@ def test_download_traefik(tmpdir): assert (traefik_bin.stat().mode & 0o777) == 0o755 +def _read_toml(path): + """Read a toml file + + print config for debugging on failure + """ + print(path) + with open(path) as f: + toml_cfg = f.read() + print(toml_cfg) + return toml.loads(toml_cfg) + + +def _read_static_config(state_dir): + return _read_toml(os.path.join(state_dir, "traefik.toml")) + + +def _read_dynamic_config(state_dir): + return _read_toml(os.path.join(state_dir, "rules", "dynamic.toml")) + + def test_default_config(tmpdir, tljh_dir): state_dir = tmpdir.mkdir("state") traefik.ensure_traefik_config(str(state_dir)) assert state_dir.join("traefik.toml").exists() - traefik_toml = os.path.join(state_dir, "traefik.toml") - with open(traefik_toml) as f: - toml_cfg = f.read() - # print config for debugging on failure - print(config.CONFIG_FILE) - print(toml_cfg) - cfg = toml.loads(toml_cfg) - assert cfg["defaultEntryPoints"] == ["http"] - assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1 - # runtime generated entry, value not testable - cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""] + os.path.join(state_dir, "traefik.toml") + rules_dir = os.path.join(state_dir, "rules") + cfg = _read_static_config(state_dir) + assert cfg["api"] == {} assert cfg["entryPoints"] == { - "http": {"address": ":80"}, + "http": { + "address": ":80", + "transport": {"respondingTimeouts": {"idleTimeout": "10m"}}, + }, "auth_api": { - "address": "127.0.0.1:8099", - "auth": {"basic": {"users": [""]}}, - "whiteList": {"sourceRange": ["127.0.0.1"]}, + "address": "localhost:8099", }, } + assert cfg["providers"] == { + "providersThrottleDuration": "0s", + "file": {"directory": rules_dir, "watch": True}, + } + + dynamic_config = _read_dynamic_config(state_dir) + assert dynamic_config == {} def test_letsencrypt_config(tljh_dir): @@ -51,34 +72,55 @@ def test_letsencrypt_config(tljh_dir): config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"] ) traefik.ensure_traefik_config(str(state_dir)) - traefik_toml = os.path.join(state_dir, "traefik.toml") - with open(traefik_toml) as f: - toml_cfg = f.read() - # print config for debugging on failure - print(config.CONFIG_FILE) - print(toml_cfg) - cfg = toml.loads(toml_cfg) - assert cfg["defaultEntryPoints"] == ["http", "https"] - assert "acme" in cfg - assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1 - # runtime generated entry, value not testable - cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""] + cfg = _read_static_config(state_dir) assert cfg["entryPoints"] == { - "http": {"address": ":80", "redirect": {"entryPoint": "https"}}, - "https": {"address": ":443", "tls": {"minVersion": "VersionTLS12"}}, + "http": { + "address": ":80", + "http": { + "redirections": { + "entryPoint": { + "scheme": "https", + "to": "https", + }, + }, + }, + "transport": {"respondingTimeouts": {"idleTimeout": "10m"}}, + }, + "https": { + "address": ":443", + "http": {"tls": {"options": "default"}}, + "transport": {"respondingTimeouts": {"idleTimeout": "10m"}}, + }, "auth_api": { - "address": "127.0.0.1:8099", - "auth": {"basic": {"users": [""]}}, - "whiteList": {"sourceRange": ["127.0.0.1"]}, + "address": "localhost:8099", }, } - assert cfg["acme"] == { + assert "tls" not in cfg + + dynamic_config = _read_dynamic_config(state_dir) + + assert dynamic_config["tls"] == { + "options": {"default": {"minVersion": "VersionTLS12"}}, + "stores": { + "default": { + "defaultGeneratedCert": { + "resolver": "letsencrypt", + "domain": { + "main": "testing.jovyan.org", + "sans": [], + }, + } + } + }, + } + assert "certificateResolvers" in cfg + assert "letsencrypt" in cfg["certificateResolvers"] + + assert cfg["certificateResolvers"]["letsencrypt"]["acme"] == { "email": "fake@jupyter.org", "storage": "acme.json", - "entryPoint": "https", "httpChallenge": {"entryPoint": "http"}, - "domains": [{"main": "testing.jovyan.org"}], } @@ -88,33 +130,50 @@ def test_manual_ssl_config(tljh_dir): config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") traefik.ensure_traefik_config(str(state_dir)) - traefik_toml = os.path.join(state_dir, "traefik.toml") - with open(traefik_toml) as f: - toml_cfg = f.read() - # print config for debugging on failure - print(config.CONFIG_FILE) - print(toml_cfg) - cfg = toml.loads(toml_cfg) - assert cfg["defaultEntryPoints"] == ["http", "https"] - assert "acme" not in cfg - assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1 - # runtime generated entry, value not testable - cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""] + + cfg = _read_static_config(state_dir) + assert cfg["entryPoints"] == { - "http": {"address": ":80", "redirect": {"entryPoint": "https"}}, - "https": { - "address": ":443", - "tls": { - "minVersion": "VersionTLS12", - "certificates": [ - {"certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key"} - ], + "http": { + "address": ":80", + "http": { + "redirections": { + "entryPoint": { + "scheme": "https", + "to": "https", + }, + }, + }, + "transport": { + "respondingTimeouts": { + "idleTimeout": "10m", + } }, }, + "https": { + "address": ":443", + "http": {"tls": {"options": "default"}}, + "transport": {"respondingTimeouts": {"idleTimeout": "10m"}}, + }, "auth_api": { - "address": "127.0.0.1:8099", - "auth": {"basic": {"users": [""]}}, - "whiteList": {"sourceRange": ["127.0.0.1"]}, + "address": "localhost:8099", + }, + } + assert "tls" not in cfg + + dynamic_config = _read_dynamic_config(state_dir) + + assert "tls" in dynamic_config + + assert dynamic_config["tls"] == { + "options": {"default": {"minVersion": "VersionTLS12"}}, + "stores": { + "default": { + "defaultCertificate": { + "certFile": "/path/to/ssl.cert", + "keyFile": "/path/to/ssl.key", + } + } }, } @@ -131,18 +190,18 @@ def test_extra_config(tmpdir, tljh_dir): toml_cfg = toml.load(traefik_toml) # Make sure the defaults are what we expect - assert toml_cfg["logLevel"] == "INFO" + assert toml_cfg["log"]["level"] == "INFO" with pytest.raises(KeyError): - toml_cfg["checkNewVersion"] - assert toml_cfg["entryPoints"]["auth_api"]["address"] == "127.0.0.1:8099" + toml_cfg["api"]["dashboard"] + assert toml_cfg["entryPoints"]["auth_api"]["address"] == "localhost:8099" extra_config = { # modify existing value - "logLevel": "ERROR", - # modify existing value with multiple levels - "entryPoints": {"auth_api": {"address": "127.0.0.1:9999"}}, + "log": { + "level": "ERROR", + }, # add new setting - "checkNewVersion": False, + "api": {"dashboard": True}, } with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file: @@ -155,6 +214,5 @@ def test_extra_config(tmpdir, tljh_dir): toml_cfg = toml.load(traefik_toml) # Check that the defaults were updated by the extra config - assert toml_cfg["logLevel"] == "ERROR" - assert toml_cfg["checkNewVersion"] == False - assert toml_cfg["entryPoints"]["auth_api"]["address"] == "127.0.0.1:9999" + assert toml_cfg["log"]["level"] == "ERROR" + assert toml_cfg["api"]["dashboard"] == True diff --git a/tljh/config.py b/tljh/config.py index 4459621..60d5cc6 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -249,8 +249,11 @@ def check_hub_ready(): r = requests.get( "http://127.0.0.1:%d%s/hub/api" % (http_port, base_url), verify=False ) + if r.status_code != 200: + print(f"Hub not ready: (HTTP status {r.status_code})") return r.status_code == 200 - except: + except Exception as e: + print(f"Hub not ready: {e}") return False diff --git a/tljh/configurer.py b/tljh/configurer.py index e5c2ea2..c5ddf18 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -239,8 +239,13 @@ def update_traefik_api(c, config): """ Set traefik api endpoint credentials """ - c.TraefikTomlProxy.traefik_api_username = config["traefik_api"]["username"] - c.TraefikTomlProxy.traefik_api_password = config["traefik_api"]["password"] + c.TraefikProxy.traefik_api_username = config["traefik_api"]["username"] + c.TraefikProxy.traefik_api_password = config["traefik_api"]["password"] + https = config["https"] + if https["enabled"]: + c.TraefikProxy.traefik_entrypoint = "https" + else: + c.TraefikProxy.traefik_entrypoint = "http" def set_cull_idle_service(config): diff --git a/tljh/jupyterhub_config.py b/tljh/jupyterhub_config.py index c61c06c..7abb617 100644 --- a/tljh/jupyterhub_config.py +++ b/tljh/jupyterhub_config.py @@ -5,13 +5,12 @@ JupyterHub config for the littlest jupyterhub. import os from glob import glob -from jupyterhub_traefik_proxy import TraefikTomlProxy - from tljh import configurer from tljh.config import CONFIG_DIR, INSTALL_PREFIX, USER_ENV_PREFIX from tljh.user_creating_spawner import UserCreatingSpawner from tljh.utils import get_plugin_manager +c = get_config() # noqa c.JupyterHub.spawner_class = UserCreatingSpawner # leave users running when the Hub restarts @@ -20,11 +19,11 @@ c.JupyterHub.cleanup_servers = False # Use a high port so users can try this on machines with a JupyterHub already present c.JupyterHub.hub_port = 15001 -c.TraefikTomlProxy.should_start = False +c.TraefikProxy.should_start = False dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, "state", "rules", "rules.toml") -c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path -c.JupyterHub.proxy_class = TraefikTomlProxy +c.TraefikFileProviderProxy.dynamic_config_file = dynamic_conf_file_path +c.JupyterHub.proxy_class = "traefik_file" c.SystemdSpawner.extra_paths = [os.path.join(USER_ENV_PREFIX, "bin")] c.SystemdSpawner.default_shell = "/bin/bash" diff --git a/tljh/traefik-dynamic.toml.tpl b/tljh/traefik-dynamic.toml.tpl new file mode 100644 index 0000000..b7e96d2 --- /dev/null +++ b/tljh/traefik-dynamic.toml.tpl @@ -0,0 +1,25 @@ +# traefik.toml dynamic config (mostly TLS) +# dynamic config in the static config file will be ignored +{% if https['enabled'] %} +[tls] + [tls.options.default] + minVersion = "VersionTLS12" + + {% if https['tls']['cert'] %} + [tls.stores.default.defaultCertificate] + certFile = "{{ https['tls']['cert'] }}" + keyFile = "{{ https['tls']['key'] }}" + {% endif %} + + {% if https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} + [tls.stores.default.defaultGeneratedCert] + resolver = "letsencrypt" + [tls.stores.default.defaultGeneratedCert.domain] + main = "{{ https['letsencrypt']['domains'][0] }}" + sans = [ + {% for domain in https['letsencrypt']['domains'][1:] %} + "{{ domain }}", + {% endfor %} + ] + {% endif %} +{% endif %} diff --git a/tljh/traefik.py b/tljh/traefik.py index e5aae6e..ca2b2b8 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -5,13 +5,13 @@ import logging import os import tarfile from glob import glob +from pathlib import Path from subprocess import run import backoff import requests import toml from jinja2 import Template -from passlib.apache import HtpasswdFile from tljh.configurer import _merge_dictionaries, load_config @@ -35,6 +35,8 @@ checksums = { "linux_arm64": "0a65ead411307669916ba629fa13f698acda0b2c5387abe0309b43e168e4e57f", } +_tljh_path = Path(__file__).parent.resolve() + def checksum_file(path_or_file): """Compute the sha256 checksum of a path""" @@ -58,11 +60,14 @@ def check_traefik_version(traefik_bin): """Check the traefik version from `traefik version` output""" try: - version_out = run([traefik_bin, "version"], capture=True, text=True) + version_out = run( + [traefik_bin, "version"], + capture_output=True, + text=True, + ).stdout except (FileNotFoundError, OSError) as e: logger.debug(f"Failed to get traefik version: {e}") return False - versions = {} for line in version_out.splitlines(): before, _, after = line.partition(":") key = before.strip() @@ -118,15 +123,6 @@ def ensure_traefik_binary(prefix): os.chmod(traefik_bin, 0o755) -def compute_basic_auth(username, password): - """Generate hashed HTTP basic auth from traefik_api username+password""" - ht = HtpasswdFile() - # generate htpassword - ht.set_password(username, password) - hashed_password = str(ht.to_string()).split(":")[1][:-3] - return username + ":" + hashed_password - - def load_extra_config(extra_config_dir): extra_configs = sorted(glob(os.path.join(extra_config_dir, "*.toml"))) # Load the toml list of files into dicts and merge them @@ -139,16 +135,13 @@ def ensure_traefik_config(state_dir): traefik_std_config_file = os.path.join(state_dir, "traefik.toml") traefik_extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") traefik_dynamic_config_dir = os.path.join(state_dir, "rules") - - config = load_config() - config["traefik_api"]["basic_auth"] = compute_basic_auth( - config["traefik_api"]["username"], - config["traefik_api"]["password"], + traefik_dynamic_config_file = os.path.join( + traefik_dynamic_config_dir, "dynamic.toml" ) - with open(os.path.join(os.path.dirname(__file__), "traefik.toml.tpl")) as f: - template = Template(f.read()) - std_config = template.render(config) + config = load_config() + config["traefik_dynamic_config_dir"] = traefik_dynamic_config_dir + https = config["https"] letsencrypt = https["letsencrypt"] tls = https["tls"] @@ -163,6 +156,14 @@ def ensure_traefik_config(state_dir): ): raise ValueError("Both email and domains must be set for letsencrypt") + with (_tljh_path / "traefik.toml.tpl").open() as f: + template = Template(f.read()) + std_config = template.render(config) + + with (_tljh_path / "traefik-dynamic.toml.tpl").open() as f: + dynamic_template = Template(f.read()) + dynamic_config = dynamic_template.render(config) + # Ensure traefik extra static config dir exists and is private os.makedirs(traefik_extra_config_dir, mode=0o700, exist_ok=True) @@ -181,6 +182,12 @@ def ensure_traefik_config(state_dir): os.fchmod(f.fileno(), 0o600) toml.dump(new_toml, f) + with open(os.path.join(traefik_dynamic_config_dir, "dynamic.toml"), "w") as f: + os.fchmod(f.fileno(), 0o600) + # validate toml syntax before writing + toml.loads(dynamic_config) + f.write(dynamic_config) + with open(os.path.join(traefik_dynamic_config_dir, "rules.toml"), "w") as f: os.fchmod(f.fileno(), 0o600) diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index 4364c16..eb6a8ee 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -1,74 +1,59 @@ -# traefik.toml file template -{% if https['enabled'] %} -defaultEntryPoints = ["http", "https"] -{% else %} -defaultEntryPoints = ["http"] -{% endif %} +# traefik.toml static config file template +# dynamic config (e.g. TLS) goes in traefik-dynamic.toml.tpl + +# enable API +[api] + +[log] +level = "INFO" -logLevel = "INFO" # log errors, which could be proxy errors [accessLog] format = "json" + [accessLog.filters] statusCodes = ["500-999"] -[accessLog.fields.headers] [accessLog.fields.headers.names] Authorization = "redact" Cookie = "redact" Set-Cookie = "redact" X-Xsrftoken = "redact" -[respondingTimeouts] -idleTimeout = "10m0s" - [entryPoints] [entryPoints.http] - address = ":{{http['port']}}" - {% if https['enabled'] %} - [entryPoints.http.redirect] - entryPoint = "https" - {% endif %} + address = ":{{ http['port'] }}" + [entryPoints.http.transport.respondingTimeouts] + idleTimeout = "10m" {% if https['enabled'] %} + [entryPoints.http.http.redirections.entryPoint] + to = "https" + scheme = "https" + [entryPoints.https] - address = ":{{https['port']}}" - [entryPoints.https.tls] - minVersion = "VersionTLS12" - {% if https['tls']['cert'] %} - [[entryPoints.https.tls.certificates]] - certFile = "{{https['tls']['cert']}}" - keyFile = "{{https['tls']['key']}}" - {% endif %} + address = ":{{ https['port'] }}" + [entryPoints.https.http.tls] + options = "default" + + [entryPoints.https.transport.respondingTimeouts] + idleTimeout = "10m" {% endif %} + [entryPoints.auth_api] - address = "127.0.0.1:{{traefik_api['port']}}" - [entryPoints.auth_api.whiteList] - sourceRange = ['{{traefik_api['ip']}}'] - [entryPoints.auth_api.auth.basic] - users = ['{{ traefik_api['basic_auth'] }}'] + address = "localhost:{{ traefik_api['port'] }}" -[wss] -protocol = "http" - -[api] -dashboard = true -entrypoint = "auth_api" - -{% if https['enabled'] and https['letsencrypt']['email'] %} -[acme] -email = "{{https['letsencrypt']['email']}}" +{% if https['enabled'] and https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} +[certificateResolvers.letsencrypt.acme] +email = "{{ https['letsencrypt']['email'] }}" storage = "acme.json" -entryPoint = "https" - [acme.httpChallenge] - entryPoint = "http" - -{% for domain in https['letsencrypt']['domains'] %} -[[acme.domains]] - main = "{{domain}}" -{% endfor %} +[certificateResolvers.letsencrypt.acme.httpChallenge] +entryPoint = "http" {% endif %} -[file] -directory = "rules" +[providers] +providersThrottleDuration = "0s" + +[providers.file] +directory = "{{ traefik_dynamic_config_dir }}" watch = true From 776ff5273b553d61cde4fa29365ae39ae1ab806b Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 13:05:34 +0200 Subject: [PATCH 03/12] update letsEncrypt config after testing verified this works now --- tests/test_traefik.py | 8 ++++---- tljh/traefik-dynamic.toml.tpl | 12 ++++++------ tljh/traefik.toml.tpl | 5 ++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index f78898f..472b3b8 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -114,13 +114,13 @@ def test_letsencrypt_config(tljh_dir): } }, } - assert "certificateResolvers" in cfg - assert "letsencrypt" in cfg["certificateResolvers"] + assert "certificatesResolvers" in cfg + assert "letsencrypt" in cfg["certificatesResolvers"] - assert cfg["certificateResolvers"]["letsencrypt"]["acme"] == { + assert cfg["certificatesResolvers"]["letsencrypt"]["acme"] == { "email": "fake@jupyter.org", "storage": "acme.json", - "httpChallenge": {"entryPoint": "http"}, + "tlsChallenge": {}, } diff --git a/tljh/traefik-dynamic.toml.tpl b/tljh/traefik-dynamic.toml.tpl index b7e96d2..a233f1d 100644 --- a/tljh/traefik-dynamic.toml.tpl +++ b/tljh/traefik-dynamic.toml.tpl @@ -5,21 +5,21 @@ [tls.options.default] minVersion = "VersionTLS12" - {% if https['tls']['cert'] %} + {% if https['tls']['cert'] -%} [tls.stores.default.defaultCertificate] certFile = "{{ https['tls']['cert'] }}" keyFile = "{{ https['tls']['key'] }}" - {% endif %} + {%- endif %} - {% if https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} + {% if https['letsencrypt']['email'] and https['letsencrypt']['domains'] -%} [tls.stores.default.defaultGeneratedCert] resolver = "letsencrypt" [tls.stores.default.defaultGeneratedCert.domain] main = "{{ https['letsencrypt']['domains'][0] }}" sans = [ - {% for domain in https['letsencrypt']['domains'][1:] %} + {% for domain in https['letsencrypt']['domains'][1:] -%} "{{ domain }}", - {% endfor %} + {%- endfor %} ] - {% endif %} + {%- endif %} {% endif %} diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index eb6a8ee..0c4ac8c 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -44,11 +44,10 @@ X-Xsrftoken = "redact" address = "localhost:{{ traefik_api['port'] }}" {% if https['enabled'] and https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} -[certificateResolvers.letsencrypt.acme] +[certificatesResolvers.letsencrypt.acme] email = "{{ https['letsencrypt']['email'] }}" storage = "acme.json" -[certificateResolvers.letsencrypt.acme.httpChallenge] -entryPoint = "http" +[certificatesResolvers.letsencrypt.acme.tlsChallenge] {% endif %} [providers] From 0e76f6dff90c745f684a93a59829a99f1e8fc9d3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 13:11:04 +0200 Subject: [PATCH 04/12] support letsencrypt staging server for easier testing --- tljh/configurer.py | 1 + tljh/traefik.toml.tpl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tljh/configurer.py b/tljh/configurer.py index c5ddf18..8e49d75 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -40,6 +40,7 @@ default = { "letsencrypt": { "email": "", "domains": [], + "staging": False, }, }, "traefik_api": { diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index 0c4ac8c..e1f82a1 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -47,6 +47,9 @@ X-Xsrftoken = "redact" [certificatesResolvers.letsencrypt.acme] email = "{{ https['letsencrypt']['email'] }}" storage = "acme.json" +{% if https['letsencrypt']['staging'] -%} +caServer = "https://acme-staging-v02.api.letsencrypt.org/directory" +{%- endif %} [certificatesResolvers.letsencrypt.acme.tlsChallenge] {% endif %} From 5763758fa40c65cf5b7b9b54fbe1f513f9818b49 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 14:53:27 +0200 Subject: [PATCH 05/12] traefik v2.10.1 Co-authored-by: Erik Sundell --- tljh/traefik.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tljh/traefik.py b/tljh/traefik.py index ca2b2b8..be07cf4 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -27,7 +27,8 @@ elif machine == "x86_64": else: plat = None -traefik_version = "2.9.9" +# Traefik releases: https://github.com/traefik/traefik/releases +traefik_version = "2.10.1" # record sha256 hashes for supported platforms here checksums = { From 7f53a4f14c6c95421775dfac710729c5e425e3fb Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 14:57:39 +0200 Subject: [PATCH 06/12] update traefik checksums --- tljh/traefik.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tljh/traefik.py b/tljh/traefik.py index be07cf4..4ea0d49 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -31,9 +31,10 @@ else: traefik_version = "2.10.1" # record sha256 hashes for supported platforms here +# checksums are published in the checksums.txt of each release checksums = { - "linux_amd64": "141db1434ae76890915486a4bc5ecf3dbafc8ece78984ce1a8db07737c42db88", - "linux_arm64": "0a65ead411307669916ba629fa13f698acda0b2c5387abe0309b43e168e4e57f", + "linux_amd64": "8d9bce0e6a5bf40b5399dbb1d5e3e5c57b9f9f04dd56a2dd57cb0713130bc824", + "linux_arm64": "260a574105e44901f8c9c562055936d81fbd9c96a21daaa575502dc69bfe390a", } _tljh_path = Path(__file__).parent.resolve() @@ -103,7 +104,7 @@ def ensure_traefik_binary(prefix): os.remove(traefik_bin) traefik_url = ( - "https://github.com/containous/traefik/releases" + "https://github.com/traefik/traefik/releases" f"/download/v{traefik_version}/traefik_v{traefik_version}_{plat}.tar.gz" ) From 33ac7239fe9900b698c50a8e40ee23403b2023d1 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 20:58:53 +0200 Subject: [PATCH 07/12] jupyterhub-traefik-proxy 1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 255ab75..7ba44c7 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( "backoff", "requests", "bcrypt", - "jupyterhub-traefik-proxy==1.0.0b3", + "jupyterhub-traefik-proxy==1.*", ], entry_points={ "console_scripts": [ From dfc8b5557cff2402c72a55183c3dcc1a4c1c0920 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 21:07:20 +0200 Subject: [PATCH 08/12] debugging progress page output --- integration-tests/test_bootstrap.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 2d4e806..d8517ec 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -4,6 +4,7 @@ Test running bootstrap script in different circumstances import concurrent.futures import os import subprocess +import sys import time BASE_IMAGE = os.getenv("BASE_IMAGE", "ubuntu:20.04") @@ -162,8 +163,13 @@ def verify_progress_page(expected_status_code, timeout): if b"HTTP/1.0 200 OK" in resp: progress_page_status = True break - except Exception: - time.sleep(2) + else: + print( + f"Unexpected progress page response: {resp[:100]}", file=sys.stderr + ) + except Exception as e: + print(f"Error getting progress page: {e}", file=sys.stderr) + time.sleep(1) continue return progress_page_status From 59648b79d49c67e10eb301af8f6ffc37058c5652 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 21:10:44 +0200 Subject: [PATCH 09/12] increase progress page test timeout seems to be why it fails sometimes --- integration-tests/test_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index d8517ec..545ba4c 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -185,7 +185,7 @@ def test_progress_page(): ) # Check if progress page started - started = verify_progress_page(expected_status_code=200, timeout=120) + started = verify_progress_page(expected_status_code=200, timeout=180) assert started # This will fail start tljh but should successfully get to the point From aee707c68ca126b0040933ceac0d44426b613aeb Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 May 2023 21:12:10 +0200 Subject: [PATCH 10/12] Specify tls cipher suites Co-authored-by: Mridul Seth --- tljh/traefik-dynamic.toml.tpl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tljh/traefik-dynamic.toml.tpl b/tljh/traefik-dynamic.toml.tpl index a233f1d..a98e7d0 100644 --- a/tljh/traefik-dynamic.toml.tpl +++ b/tljh/traefik-dynamic.toml.tpl @@ -4,7 +4,14 @@ [tls] [tls.options.default] minVersion = "VersionTLS12" - + cipherSuites = [ + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + ] {% if https['tls']['cert'] -%} [tls.stores.default.defaultCertificate] certFile = "{{ https['tls']['cert'] }}" From c5dec7f3b4511c059dc6921a8e57a55b178b6534 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 17 May 2023 09:41:55 +0200 Subject: [PATCH 11/12] update test expectations for traefik ssl --- tests/test_traefik.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index 472b3b8..4098586 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -101,7 +101,19 @@ def test_letsencrypt_config(tljh_dir): dynamic_config = _read_dynamic_config(state_dir) assert dynamic_config["tls"] == { - "options": {"default": {"minVersion": "VersionTLS12"}}, + "options": { + "default": { + "minVersion": "VersionTLS12", + "cipherSuites": [ + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + ], + } + }, "stores": { "default": { "defaultGeneratedCert": { @@ -166,7 +178,19 @@ def test_manual_ssl_config(tljh_dir): assert "tls" in dynamic_config assert dynamic_config["tls"] == { - "options": {"default": {"minVersion": "VersionTLS12"}}, + "options": { + "default": { + "minVersion": "VersionTLS12", + "cipherSuites": [ + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + ], + }, + }, "stores": { "default": { "defaultCertificate": { From 324ded0003c0f0ae38188219ea8c5c95bec2bfe3 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 11:18:09 +0200 Subject: [PATCH 12/12] Consistent whitespace chomping in jinja templates Always chomp left, never right. This is what we do in the helm chart templates and has been very easy to be consistent with for a good result with little to no issues. --- tljh/traefik-dynamic.toml.tpl | 8 ++++---- tljh/traefik.toml.tpl | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tljh/traefik-dynamic.toml.tpl b/tljh/traefik-dynamic.toml.tpl index a98e7d0..f1144d6 100644 --- a/tljh/traefik-dynamic.toml.tpl +++ b/tljh/traefik-dynamic.toml.tpl @@ -1,6 +1,6 @@ # traefik.toml dynamic config (mostly TLS) # dynamic config in the static config file will be ignored -{% if https['enabled'] %} +{%- if https['enabled'] %} [tls] [tls.options.default] minVersion = "VersionTLS12" @@ -12,13 +12,13 @@ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", ] - {% if https['tls']['cert'] -%} + {%- if https['tls']['cert'] %} [tls.stores.default.defaultCertificate] certFile = "{{ https['tls']['cert'] }}" keyFile = "{{ https['tls']['key'] }}" {%- endif %} - {% if https['letsencrypt']['email'] and https['letsencrypt']['domains'] -%} + {%- if https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} [tls.stores.default.defaultGeneratedCert] resolver = "letsencrypt" [tls.stores.default.defaultGeneratedCert.domain] @@ -29,4 +29,4 @@ {%- endfor %} ] {%- endif %} -{% endif %} +{%- endif %} diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index e1f82a1..fa5b6ef 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -23,35 +23,37 @@ X-Xsrftoken = "redact" [entryPoints] [entryPoints.http] address = ":{{ http['port'] }}" + [entryPoints.http.transport.respondingTimeouts] idleTimeout = "10m" - {% if https['enabled'] %} + {%- if https['enabled'] %} [entryPoints.http.http.redirections.entryPoint] to = "https" scheme = "https" [entryPoints.https] address = ":{{ https['port'] }}" + [entryPoints.https.http.tls] options = "default" [entryPoints.https.transport.respondingTimeouts] idleTimeout = "10m" - {% endif %} + {%- endif %} [entryPoints.auth_api] address = "localhost:{{ traefik_api['port'] }}" -{% if https['enabled'] and https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} +{%- if https['enabled'] and https['letsencrypt']['email'] and https['letsencrypt']['domains'] %} [certificatesResolvers.letsencrypt.acme] email = "{{ https['letsencrypt']['email'] }}" storage = "acme.json" -{% if https['letsencrypt']['staging'] -%} +{%- if https['letsencrypt']['staging'] %} caServer = "https://acme-staging-v02.api.letsencrypt.org/directory" {%- endif %} [certificatesResolvers.letsencrypt.acme.tlsChallenge] -{% endif %} +{%- endif %} [providers] providersThrottleDuration = "0s"