Merge branch 'jupyterhub:main' into update-google-auth-docs

integration-tests/conftest.py

@@ -25,5 +25,5 @@ def preserve_config(request):
                 f.write(save_config)
         elif os.path.exists(CONFIG_FILE):
             os.remove(CONFIG_FILE)
-        reload_component("hub")
         reload_component("proxy")
+        reload_component("hub")

integration-tests/test_bootstrap.py

@@ -4,6 +4,7 @@ Test running bootstrap script in different circumstances
 import concurrent.futures
 import os
 import subprocess
+import sys
 import time
 
 BASE_IMAGE = os.getenv("BASE_IMAGE", "ubuntu:20.04")
@@ -162,8 +163,13 @@ def verify_progress_page(expected_status_code, timeout):
             if b"HTTP/1.0 200 OK" in resp:
                 progress_page_status = True
                 break
-        except Exception:
+            else:
-            time.sleep(2)
+                print(
+                    f"Unexpected progress page response: {resp[:100]}", file=sys.stderr
+                )
+        except Exception as e:
+            print(f"Error getting progress page: {e}", file=sys.stderr)
+            time.sleep(1)
             continue
 
     return progress_page_status
@@ -179,7 +185,7 @@ def test_progress_page():
         )
 
         # Check if progress page started
-        started = verify_progress_page(expected_status_code=200, timeout=120)
+        started = verify_progress_page(expected_status_code=200, timeout=180)
         assert started
 
         # This will fail start tljh but should successfully get to the point

integration-tests/test_proxy.py

@@ -19,9 +19,7 @@ from tljh.config import (
 
 
 def send_request(url, max_sleep, validate_cert=True, username=None, password=None):
-    resp = None
     for i in range(max_sleep):
-        time.sleep(i)
         try:
             req = HTTPRequest(
                 url,
@@ -32,12 +30,12 @@ def send_request(url, max_sleep, validate_cert=True, username=None, password=Non
                 follow_redirects=True,
                 max_redirects=15,
             )
-            resp = HTTPClient().fetch(req)
+            return HTTPClient().fetch(req)
-            break
         except Exception as e:
+            if i + 1 == max_sleep:
+                raise
             print(e)
-
+            time.sleep(i)
-    return resp
 
 
 def test_manual_https(preserve_config):
@@ -104,37 +102,51 @@ def test_extra_traefik_config():
     os.makedirs(dynamic_config_dir, exist_ok=True)
 
     extra_static_config = {
-        "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}},
+        "entryPoints": {"alsoHub": {"address": "127.0.0.1:9999"}},
-        "api": {"dashboard": True, "entrypoint": "no_auth_api"},
     }
 
     extra_dynamic_config = {
-        "frontends": {
+        "http": {
-            "test": {
+            "middlewares": {
-                "backend": "test",
+                "testHubStripPrefix": {
-                "routes": {
+                    "stripPrefix": {"prefixes": ["/the/hub/runs/here/too"]}
-                    "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"}
+                }
+            },
+            "routers": {
+                "test1": {
+                    "rule": "PathPrefix(`/hub`)",
+                    "entryPoints": ["alsoHub"],
+                    "service": "test",
                 },
-            }
+                "test2": {
-        },
+                    "rule": "PathPrefix(`/the/hub/runs/here/too`)",
-        "backends": {
+                    "middlewares": ["testHubStripPrefix"],
-            # redirect to hub
+                    "entryPoints": ["http"],
-            "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}}
+                    "service": "test",
+                },
+            },
+            "services": {
+                "test": {
+                    "loadBalancer": {
+                        # forward requests to the hub
+                        "servers": [{"url": "http://127.0.0.1:15001"}]
+                    }
+                }
+            },
         },
     }
 
     success = False
     for i in range(5):
-        time.sleep(i)
         try:
             with pytest.raises(HTTPClientError, match="HTTP 401: Unauthorized"):
-                # The default dashboard entrypoint requires authentication, so it should fail
+                # The default api entrypoint requires authentication, so it should fail
-                req = HTTPRequest("http://127.0.0.1:8099/dashboard/", method="GET")
+                HTTPClient().fetch("http://localhost:8099/api")
-                HTTPClient().fetch(req)
             success = True
             break
-        except Exception:
+        except Exception as e:
-            pass
+            print(e)
+            time.sleep(i)
 
     assert success == True
 
@@ -153,8 +165,9 @@ def test_extra_traefik_config():
     # load the extra config
     reload_component("proxy")
 
+    # check hub page
     # the new dashboard entrypoint shouldn't require authentication anymore
-    resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5)
+    resp = send_request(url="http://127.0.0.1:9999/hub/login", max_sleep=5)
     assert resp.code == 200
 
     # test extra dynamic config

setup.py

@@ -14,11 +14,10 @@ setup(
         "ruamel.yaml==0.17.*",
         "jinja2",
         "pluggy==1.*",
-        "passlib",
         "backoff",
         "requests",
         "bcrypt",
-        "jupyterhub-traefik-proxy==0.3.*",
+        "jupyterhub-traefik-proxy==1.*",
     ],
     entry_points={
         "console_scripts": [

tests/test_configurer.py

@@ -156,8 +156,8 @@ def test_traefik_api_default():
     """
     c = apply_mock_config({})
 
-    assert c.TraefikTomlProxy.traefik_api_username == "api_admin"
+    assert c.TraefikProxy.traefik_api_username == "api_admin"
-    assert len(c.TraefikTomlProxy.traefik_api_password) == 0
+    assert len(c.TraefikProxy.traefik_api_password) == 0
 
 
 def test_set_traefik_api():
@@ -167,8 +167,8 @@ def test_set_traefik_api():
     c = apply_mock_config(
         {"traefik_api": {"username": "some_user", "password": "1234"}}
     )
-    assert c.TraefikTomlProxy.traefik_api_username == "some_user"
+    assert c.TraefikProxy.traefik_api_username == "some_user"
-    assert c.TraefikTomlProxy.traefik_api_password == "1234"
+    assert c.TraefikProxy.traefik_api_password == "1234"
 
 
 def test_cull_service_default():
@@ -268,7 +268,7 @@ def test_load_secrets(tljh_dir):
     tljh_config = configurer.load_config()
     assert tljh_config["traefik_api"]["password"] == "traefik-password"
     c = apply_mock_config(tljh_config)
-    assert c.TraefikTomlProxy.traefik_api_password == "traefik-password"
+    assert c.TraefikProxy.traefik_api_password == "traefik-password"
 
 
 def test_auth_native():

tests/test_traefik.py

@@ -15,30 +15,51 @@ def test_download_traefik(tmpdir):
     assert (traefik_bin.stat().mode & 0o777) == 0o755
 
 
+def _read_toml(path):
+    """Read a toml file
+
+    print config for debugging on failure
+    """
+    print(path)
+    with open(path) as f:
+        toml_cfg = f.read()
+        print(toml_cfg)
+        return toml.loads(toml_cfg)
+
+
+def _read_static_config(state_dir):
+    return _read_toml(os.path.join(state_dir, "traefik.toml"))
+
+
+def _read_dynamic_config(state_dir):
+    return _read_toml(os.path.join(state_dir, "rules", "dynamic.toml"))
+
+
 def test_default_config(tmpdir, tljh_dir):
     state_dir = tmpdir.mkdir("state")
     traefik.ensure_traefik_config(str(state_dir))
     assert state_dir.join("traefik.toml").exists()
-    traefik_toml = os.path.join(state_dir, "traefik.toml")
+    os.path.join(state_dir, "traefik.toml")
-    with open(traefik_toml) as f:
+    rules_dir = os.path.join(state_dir, "rules")
-        toml_cfg = f.read()
-        # print config for debugging on failure
-        print(config.CONFIG_FILE)
-        print(toml_cfg)
-        cfg = toml.loads(toml_cfg)
-    assert cfg["defaultEntryPoints"] == ["http"]
-    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
-    # runtime generated entry, value not testable
-    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
 
+    cfg = _read_static_config(state_dir)
+    assert cfg["api"] == {}
     assert cfg["entryPoints"] == {
-        "http": {"address": ":80"},
+        "http": {
+            "address": ":80",
+            "transport": {"respondingTimeouts": {"idleTimeout": "10m"}},
+        },
         "auth_api": {
-            "address": "127.0.0.1:8099",
+            "address": "localhost:8099",
-            "auth": {"basic": {"users": [""]}},
-            "whiteList": {"sourceRange": ["127.0.0.1"]},
         },
     }
+    assert cfg["providers"] == {
+        "providersThrottleDuration": "0s",
+        "file": {"directory": rules_dir, "watch": True},
+    }
+
+    dynamic_config = _read_dynamic_config(state_dir)
+    assert dynamic_config == {}
 
 
 def test_letsencrypt_config(tljh_dir):
@@ -51,34 +72,67 @@ def test_letsencrypt_config(tljh_dir):
         config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"]
     )
     traefik.ensure_traefik_config(str(state_dir))
-    traefik_toml = os.path.join(state_dir, "traefik.toml")
-    with open(traefik_toml) as f:
-        toml_cfg = f.read()
-        # print config for debugging on failure
-        print(config.CONFIG_FILE)
-        print(toml_cfg)
-        cfg = toml.loads(toml_cfg)
-    assert cfg["defaultEntryPoints"] == ["http", "https"]
-    assert "acme" in cfg
-    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
-    # runtime generated entry, value not testable
-    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
 
+    cfg = _read_static_config(state_dir)
     assert cfg["entryPoints"] == {
-        "http": {"address": ":80", "redirect": {"entryPoint": "https"}},
+        "http": {
-        "https": {"address": ":443", "tls": {"minVersion": "VersionTLS12"}},
+            "address": ":80",
+            "http": {
+                "redirections": {
+                    "entryPoint": {
+                        "scheme": "https",
+                        "to": "https",
+                    },
+                },
+            },
+            "transport": {"respondingTimeouts": {"idleTimeout": "10m"}},
+        },
+        "https": {
+            "address": ":443",
+            "http": {"tls": {"options": "default"}},
+            "transport": {"respondingTimeouts": {"idleTimeout": "10m"}},
+        },
         "auth_api": {
-            "address": "127.0.0.1:8099",
+            "address": "localhost:8099",
-            "auth": {"basic": {"users": [""]}},
-            "whiteList": {"sourceRange": ["127.0.0.1"]},
         },
     }
-    assert cfg["acme"] == {
+    assert "tls" not in cfg
+
+    dynamic_config = _read_dynamic_config(state_dir)
+
+    assert dynamic_config["tls"] == {
+        "options": {
+            "default": {
+                "minVersion": "VersionTLS12",
+                "cipherSuites": [
+                    "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+                    "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+                    "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+                    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
+                ],
+            }
+        },
+        "stores": {
+            "default": {
+                "defaultGeneratedCert": {
+                    "resolver": "letsencrypt",
+                    "domain": {
+                        "main": "testing.jovyan.org",
+                        "sans": [],
+                    },
+                }
+            }
+        },
+    }
+    assert "certificatesResolvers" in cfg
+    assert "letsencrypt" in cfg["certificatesResolvers"]
+
+    assert cfg["certificatesResolvers"]["letsencrypt"]["acme"] == {
         "email": "fake@jupyter.org",
         "storage": "acme.json",
-        "entryPoint": "https",
+        "tlsChallenge": {},
-        "httpChallenge": {"entryPoint": "http"},
-        "domains": [{"main": "testing.jovyan.org"}],
     }
 
 
@@ -88,33 +142,62 @@ def test_manual_ssl_config(tljh_dir):
     config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key")
     config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert")
     traefik.ensure_traefik_config(str(state_dir))
-    traefik_toml = os.path.join(state_dir, "traefik.toml")
+
-    with open(traefik_toml) as f:
+    cfg = _read_static_config(state_dir)
-        toml_cfg = f.read()
+
-        # print config for debugging on failure
-        print(config.CONFIG_FILE)
-        print(toml_cfg)
-        cfg = toml.loads(toml_cfg)
-    assert cfg["defaultEntryPoints"] == ["http", "https"]
-    assert "acme" not in cfg
-    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
-    # runtime generated entry, value not testable
-    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
     assert cfg["entryPoints"] == {
-        "http": {"address": ":80", "redirect": {"entryPoint": "https"}},
+        "http": {
+            "address": ":80",
+            "http": {
+                "redirections": {
+                    "entryPoint": {
+                        "scheme": "https",
+                        "to": "https",
+                    },
+                },
+            },
+            "transport": {
+                "respondingTimeouts": {
+                    "idleTimeout": "10m",
+                }
+            },
+        },
         "https": {
             "address": ":443",
-            "tls": {
+            "http": {"tls": {"options": "default"}},
+            "transport": {"respondingTimeouts": {"idleTimeout": "10m"}},
+        },
+        "auth_api": {
+            "address": "localhost:8099",
+        },
+    }
+    assert "tls" not in cfg
+
+    dynamic_config = _read_dynamic_config(state_dir)
+
+    assert "tls" in dynamic_config
+
+    assert dynamic_config["tls"] == {
+        "options": {
+            "default": {
                 "minVersion": "VersionTLS12",
-                "certificates": [
+                "cipherSuites": [
-                    {"certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key"}
+                    "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+                    "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+                    "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+                    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
                 ],
             },
         },
-        "auth_api": {
+        "stores": {
-            "address": "127.0.0.1:8099",
+            "default": {
-            "auth": {"basic": {"users": [""]}},
+                "defaultCertificate": {
-            "whiteList": {"sourceRange": ["127.0.0.1"]},
+                    "certFile": "/path/to/ssl.cert",
+                    "keyFile": "/path/to/ssl.key",
+                }
+            }
         },
     }
 
@@ -131,18 +214,18 @@ def test_extra_config(tmpdir, tljh_dir):
     toml_cfg = toml.load(traefik_toml)
 
     # Make sure the defaults are what we expect
-    assert toml_cfg["logLevel"] == "INFO"
+    assert toml_cfg["log"]["level"] == "INFO"
     with pytest.raises(KeyError):
-        toml_cfg["checkNewVersion"]
+        toml_cfg["api"]["dashboard"]
-    assert toml_cfg["entryPoints"]["auth_api"]["address"] == "127.0.0.1:8099"
+    assert toml_cfg["entryPoints"]["auth_api"]["address"] == "localhost:8099"
 
     extra_config = {
         # modify existing value
-        "logLevel": "ERROR",
+        "log": {
-        # modify existing value with multiple levels
+            "level": "ERROR",
-        "entryPoints": {"auth_api": {"address": "127.0.0.1:9999"}},
+        },
         # add new setting
-        "checkNewVersion": False,
+        "api": {"dashboard": True},
     }
 
     with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file:
@@ -155,6 +238,5 @@ def test_extra_config(tmpdir, tljh_dir):
     toml_cfg = toml.load(traefik_toml)
 
     # Check that the defaults were updated by the extra config
-    assert toml_cfg["logLevel"] == "ERROR"
+    assert toml_cfg["log"]["level"] == "ERROR"
-    assert toml_cfg["checkNewVersion"] == False
+    assert toml_cfg["api"]["dashboard"] == True
-    assert toml_cfg["entryPoints"]["auth_api"]["address"] == "127.0.0.1:9999"

tljh/config.py

@@ -249,8 +249,11 @@ def check_hub_ready():
         r = requests.get(
             "http://127.0.0.1:%d%s/hub/api" % (http_port, base_url), verify=False
         )
+        if r.status_code != 200:
+            print(f"Hub not ready: (HTTP status {r.status_code})")
         return r.status_code == 200
-    except:
+    except Exception as e:
+        print(f"Hub not ready: {e}")
         return False
 
 

tljh/configurer.py

@@ -40,6 +40,7 @@ default = {
         "letsencrypt": {
             "email": "",
             "domains": [],
+            "staging": False,
         },
     },
     "traefik_api": {
@@ -239,8 +240,13 @@ def update_traefik_api(c, config):
     """
     Set traefik api endpoint credentials
     """
-    c.TraefikTomlProxy.traefik_api_username = config["traefik_api"]["username"]
+    c.TraefikProxy.traefik_api_username = config["traefik_api"]["username"]
-    c.TraefikTomlProxy.traefik_api_password = config["traefik_api"]["password"]
+    c.TraefikProxy.traefik_api_password = config["traefik_api"]["password"]
+    https = config["https"]
+    if https["enabled"]:
+        c.TraefikProxy.traefik_entrypoint = "https"
+    else:
+        c.TraefikProxy.traefik_entrypoint = "http"
 
 
 def set_cull_idle_service(config):

tljh/jupyterhub_config.py

@@ -5,13 +5,12 @@ JupyterHub config for the littlest jupyterhub.
 import os
 from glob import glob
 
-from jupyterhub_traefik_proxy import TraefikTomlProxy
-
 from tljh import configurer
 from tljh.config import CONFIG_DIR, INSTALL_PREFIX, USER_ENV_PREFIX
 from tljh.user_creating_spawner import UserCreatingSpawner
 from tljh.utils import get_plugin_manager
 
+c = get_config()  # noqa
 c.JupyterHub.spawner_class = UserCreatingSpawner
 
 # leave users running when the Hub restarts
@@ -20,11 +19,11 @@ c.JupyterHub.cleanup_servers = False
 # Use a high port so users can try this on machines with a JupyterHub already present
 c.JupyterHub.hub_port = 15001
 
-c.TraefikTomlProxy.should_start = False
+c.TraefikProxy.should_start = False
 
 dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, "state", "rules", "rules.toml")
-c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path
+c.TraefikFileProviderProxy.dynamic_config_file = dynamic_conf_file_path
-c.JupyterHub.proxy_class = TraefikTomlProxy
+c.JupyterHub.proxy_class = "traefik_file"
 
 c.SystemdSpawner.extra_paths = [os.path.join(USER_ENV_PREFIX, "bin")]
 c.SystemdSpawner.default_shell = "/bin/bash"

tljh/traefik-dynamic.toml.tpl

@@ -0,0 +1,32 @@
+# traefik.toml dynamic config (mostly TLS)
+# dynamic config in the static config file will be ignored
+{%- if https['enabled'] %}
+[tls]
+  [tls.options.default]
+  minVersion = "VersionTLS12"
+  cipherSuites = [
+    "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+    "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+    "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+    "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+    "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
+  ]
+  {%- if https['tls']['cert'] %}
+  [tls.stores.default.defaultCertificate]
+    certFile = "{{ https['tls']['cert'] }}"
+    keyFile = "{{ https['tls']['key'] }}"
+  {%- endif %}
+
+  {%- if https['letsencrypt']['email'] and https['letsencrypt']['domains'] %}
+  [tls.stores.default.defaultGeneratedCert]
+  resolver = "letsencrypt"
+    [tls.stores.default.defaultGeneratedCert.domain]
+    main = "{{ https['letsencrypt']['domains'][0] }}"
+    sans = [
+      {% for domain in https['letsencrypt']['domains'][1:] -%}
+      "{{ domain }}",
+      {%- endfor %}
+    ]
+  {%- endif %}
+{%- endif %}

tljh/traefik.py

@@ -1,40 +1,53 @@
 """Traefik installation and setup"""
 import hashlib
+import io
+import logging
 import os
+import tarfile
 from glob import glob
+from pathlib import Path
+from subprocess import run
 
 import backoff
 import requests
 import toml
 from jinja2 import Template
-from passlib.apache import HtpasswdFile
 
 from tljh.configurer import _merge_dictionaries, load_config
 
 from .config import CONFIG_DIR
 
-# traefik 2.7.x is not supported yet, use v1.7.x for now
+logger = logging.getLogger("tljh")
-# see: https://github.com/jupyterhub/traefik-proxy/issues/97
+
 machine = os.uname().machine
 if machine == "aarch64":
-    plat = "linux-arm64"
+    plat = "linux_arm64"
 elif machine == "x86_64":
-    plat = "linux-amd64"
+    plat = "linux_amd64"
 else:
-    raise OSError(f"Error. Platform: {os.uname().sysname} / {machine} Not supported.")
+    plat = None
-traefik_version = "1.7.33"
+
+# Traefik releases: https://github.com/traefik/traefik/releases
+traefik_version = "2.10.1"
 
 # record sha256 hashes for supported platforms here
+# checksums are published in the checksums.txt of each release
 checksums = {
-    "linux-amd64": "314ffeaa4cd8ed6ab7b779e9b6773987819f79b23c28d7ab60ace4d3683c5935",
+    "linux_amd64": "8d9bce0e6a5bf40b5399dbb1d5e3e5c57b9f9f04dd56a2dd57cb0713130bc824",
-    "linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1",
+    "linux_arm64": "260a574105e44901f8c9c562055936d81fbd9c96a21daaa575502dc69bfe390a",
 }
 
+_tljh_path = Path(__file__).parent.resolve()
 
-def checksum_file(path):
+
+def checksum_file(path_or_file):
     """Compute the sha256 checksum of a path"""
     hasher = hashlib.sha256()
-    with open(path, "rb") as f:
+    if hasattr(path_or_file, "read"):
+        f = path_or_file
+    else:
+        f = open(path_or_file, "rb")
+    with f:
         for chunk in iter(lambda: f.read(4096), b""):
             hasher.update(chunk)
     return hasher.hexdigest()
@@ -45,48 +58,71 @@ def fatal_error(e):
     return str(e) != "ContentTooShort" and not isinstance(e, ConnectionResetError)
 
 
+def check_traefik_version(traefik_bin):
+    """Check the traefik version from `traefik version` output"""
+
+    try:
+        version_out = run(
+            [traefik_bin, "version"],
+            capture_output=True,
+            text=True,
+        ).stdout
+    except (FileNotFoundError, OSError) as e:
+        logger.debug(f"Failed to get traefik version: {e}")
+        return False
+    for line in version_out.splitlines():
+        before, _, after = line.partition(":")
+        key = before.strip()
+        if key.lower() == "version":
+            version = after.strip()
+            if version == traefik_version:
+                logger.debug(f"Found {traefik_bin} {version}")
+                return True
+            else:
+                logger.info(
+                    f"Found {traefik_bin} version {version} != {traefik_version}"
+                )
+                return False
+
+    logger.debug(f"Failed to extract traefik version from: {version_out}")
+    return False
+
+
 @backoff.on_exception(backoff.expo, Exception, max_tries=2, giveup=fatal_error)
 def ensure_traefik_binary(prefix):
     """Download and install the traefik binary to a location identified by a prefix path such as '/opt/tljh/hub/'"""
+    if plat is None:
+        raise OSError(
+            f"Error. Platform: {os.uname().sysname} / {machine} Not supported."
+        )
+    traefik_bin_dir = os.path.join(prefix, "bin")
     traefik_bin = os.path.join(prefix, "bin", "traefik")
     if os.path.exists(traefik_bin):
-        checksum = checksum_file(traefik_bin)
+        if check_traefik_version(traefik_bin):
-        if checksum == checksums[plat]:
-            # already have the right binary
-            # ensure permissions and we're done
-            os.chmod(traefik_bin, 0o755)
             return
         else:
-            print(f"checksum mismatch on {traefik_bin}")
             os.remove(traefik_bin)
 
     traefik_url = (
-        "https://github.com/containous/traefik/releases"
+        "https://github.com/traefik/traefik/releases"
-        f"/download/v{traefik_version}/traefik_{plat}"
+        f"/download/v{traefik_version}/traefik_v{traefik_version}_{plat}.tar.gz"
     )
 
-    print(f"Downloading traefik {traefik_version}...")
+    logger.info(f"Downloading traefik {traefik_version} from {traefik_url}...")
     # download the file
     response = requests.get(traefik_url)
+    response.raise_for_status()
     if response.status_code == 206:
         raise Exception("ContentTooShort")
-    with open(traefik_bin, "wb") as f:
-        f.write(response.content)
-    os.chmod(traefik_bin, 0o755)
 
     # verify that we got what we expected
-    checksum = checksum_file(traefik_bin)
+    checksum = checksum_file(io.BytesIO(response.content))
     if checksum != checksums[plat]:
-        raise OSError(f"Checksum failed {traefik_bin}: {checksum} != {checksums[plat]}")
+        raise OSError(f"Checksum failed {traefik_url}: {checksum} != {checksums[plat]}")
 
-
+    with tarfile.open(fileobj=io.BytesIO(response.content)) as tf:
-def compute_basic_auth(username, password):
+        tf.extract("traefik", path=traefik_bin_dir)
-    """Generate hashed HTTP basic auth from traefik_api username+password"""
+    os.chmod(traefik_bin, 0o755)
-    ht = HtpasswdFile()
-    # generate htpassword
-    ht.set_password(username, password)
-    hashed_password = str(ht.to_string()).split(":")[1][:-3]
-    return username + ":" + hashed_password
 
 
 def load_extra_config(extra_config_dir):
@@ -101,16 +137,13 @@ def ensure_traefik_config(state_dir):
     traefik_std_config_file = os.path.join(state_dir, "traefik.toml")
     traefik_extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d")
     traefik_dynamic_config_dir = os.path.join(state_dir, "rules")
-
+    traefik_dynamic_config_file = os.path.join(
-    config = load_config()
+        traefik_dynamic_config_dir, "dynamic.toml"
-    config["traefik_api"]["basic_auth"] = compute_basic_auth(
-        config["traefik_api"]["username"],
-        config["traefik_api"]["password"],
     )
 
-    with open(os.path.join(os.path.dirname(__file__), "traefik.toml.tpl")) as f:
+    config = load_config()
-        template = Template(f.read())
+    config["traefik_dynamic_config_dir"] = traefik_dynamic_config_dir
-    std_config = template.render(config)
+
     https = config["https"]
     letsencrypt = https["letsencrypt"]
     tls = https["tls"]
@@ -125,6 +158,14 @@ def ensure_traefik_config(state_dir):
         ):
             raise ValueError("Both email and domains must be set for letsencrypt")
 
+    with (_tljh_path / "traefik.toml.tpl").open() as f:
+        template = Template(f.read())
+    std_config = template.render(config)
+
+    with (_tljh_path / "traefik-dynamic.toml.tpl").open() as f:
+        dynamic_template = Template(f.read())
+    dynamic_config = dynamic_template.render(config)
+
     # Ensure traefik extra static config dir exists and is private
     os.makedirs(traefik_extra_config_dir, mode=0o700, exist_ok=True)
 
@@ -143,6 +184,12 @@ def ensure_traefik_config(state_dir):
         os.fchmod(f.fileno(), 0o600)
         toml.dump(new_toml, f)
 
+    with open(os.path.join(traefik_dynamic_config_dir, "dynamic.toml"), "w") as f:
+        os.fchmod(f.fileno(), 0o600)
+        # validate toml syntax before writing
+        toml.loads(dynamic_config)
+        f.write(dynamic_config)
+
     with open(os.path.join(traefik_dynamic_config_dir, "rules.toml"), "w") as f:
         os.fchmod(f.fileno(), 0o600)
 

tljh/traefik.toml.tpl

@@ -1,74 +1,63 @@
-# traefik.toml file template
+# traefik.toml static config file template
-{% if https['enabled'] %}
+# dynamic config (e.g. TLS) goes in traefik-dynamic.toml.tpl
-defaultEntryPoints = ["http", "https"]
+
-{% else %}
+# enable API
-defaultEntryPoints = ["http"]
+[api]
-{% endif %}
+
+[log]
+level = "INFO"
 
-logLevel = "INFO"
 # log errors, which could be proxy errors
 [accessLog]
 format = "json"
+
 [accessLog.filters]
 statusCodes = ["500-999"]
 
-[accessLog.fields.headers]
 [accessLog.fields.headers.names]
 Authorization = "redact"
 Cookie = "redact"
 Set-Cookie = "redact"
 X-Xsrftoken = "redact"
 
-[respondingTimeouts]
-idleTimeout = "10m0s"
-
 [entryPoints]
   [entryPoints.http]
-  address = ":{{http['port']}}"
+  address = ":{{ http['port'] }}"
-  {% if https['enabled'] %}
+
-    [entryPoints.http.redirect]
+  [entryPoints.http.transport.respondingTimeouts]
-    entryPoint = "https"
+  idleTimeout = "10m"
-  {% endif %}
+
+  {%- if https['enabled'] %}
+  [entryPoints.http.http.redirections.entryPoint]
+  to = "https"
+  scheme = "https"
 
-  {% if https['enabled'] %}
   [entryPoints.https]
-  address = ":{{https['port']}}"
+  address = ":{{ https['port'] }}"
-  [entryPoints.https.tls]
+
-  minVersion = "VersionTLS12"
+  [entryPoints.https.http.tls]
-  {% if https['tls']['cert'] %}
+  options = "default"
-    [[entryPoints.https.tls.certificates]]
+
-      certFile = "{{https['tls']['cert']}}"
+  [entryPoints.https.transport.respondingTimeouts]
-      keyFile = "{{https['tls']['key']}}"
+  idleTimeout = "10m"
-  {% endif %}
+  {%- endif %}
-  {% endif %}
+
   [entryPoints.auth_api]
-  address = "127.0.0.1:{{traefik_api['port']}}"
+  address = "localhost:{{ traefik_api['port'] }}"
-  [entryPoints.auth_api.whiteList]
-  sourceRange = ['{{traefik_api['ip']}}']
-  [entryPoints.auth_api.auth.basic]
-  users = ['{{ traefik_api['basic_auth'] }}']
 
-[wss]
+{%- if https['enabled'] and https['letsencrypt']['email'] and https['letsencrypt']['domains'] %}
-protocol = "http"
+[certificatesResolvers.letsencrypt.acme]
-
+email = "{{ https['letsencrypt']['email'] }}"
-[api]
-dashboard = true
-entrypoint = "auth_api"
-
-{% if https['enabled'] and https['letsencrypt']['email'] %}
-[acme]
-email = "{{https['letsencrypt']['email']}}"
 storage = "acme.json"
-entryPoint = "https"
+{%- if https['letsencrypt']['staging'] %}
-  [acme.httpChallenge]
+caServer = "https://acme-staging-v02.api.letsencrypt.org/directory"
-  entryPoint = "http"
+{%- endif %}
+[certificatesResolvers.letsencrypt.acme.tlsChallenge]
+{%- endif %}
 
-{% for domain in https['letsencrypt']['domains'] %}
+[providers]
-[[acme.domains]]
+providersThrottleDuration = "0s"
-  main = "{{domain}}"
-{% endfor %}
-{% endif %}
 
-[file]
+[providers.file]
-directory = "rules"
+directory = "{{ traefik_dynamic_config_dir }}"
 watch = true