diff --git a/docs/topic/escape-hatch.rst b/docs/topic/escape-hatch.rst index 9e2649f..d2fd869 100644 --- a/docs/topic/escape-hatch.rst +++ b/docs/topic/escape-hatch.rst @@ -8,11 +8,19 @@ Custom configuration snippets The two main TLJH components are **JupyterHub** and **Traefik**. * JupyterHub takes its configuration from the ``jupyterhub_config.py`` file. -* Traefik takes its configuration from the ``traefik.toml`` file. +* Traefik loads its: + * `static configuration `_ + from the ``traefik.toml`` file. + * `dynamic configuration `_ + from the ``rules`` directory. -These files are created by TLJH during installation and can be edited by the -user only through ``tljh-config``. Any direct modification to these files -is unsupported, and will cause hard to debug issues. +The ``jupyterhub_config.py`` and ``traefik.toml`` files are created by TLJH during installation +and can be edited by the user only through ``tljh-config``. The ``rules`` directory is also created +during install along with a ``rules/rules.toml`` file, to be used by JupyterHub to store the routing +table from users to their notebooks. + +.. note:: + Any direct modification to these files is unsupported, and will cause hard to debug issues. But because sometimes TLJH needs to be customized in ways that are not officially supported, an escape hatch has been introduced to allow easily extending the @@ -65,3 +73,22 @@ proxy for the new configuration to take effect: sudo tljh-config reload proxy .. warning:: This instructions might change when TLJH will switch to Traefik > 2.0 + +Extending ``rules.toml`` +======================== + +``Traefik`` is configured to load its routing table from the ``/opt/tljh/state/rules`` +directory. The existing ``rules.toml`` file inside this directory is used by +``jupyterhub-traefik-proxy`` to add the JupyterHub routes from users to their notebook servers +and shouldn't be modified. + +However, the routing table can be extended outside JupyterHub's scope using the ``rules`` +directory, by adding other dynamic configuration files with the desired routing rules. + +.. note:: + * Any files in ``/opt/tljh/state/rules`` that end in ``.toml`` will be hot reload by Traefik. + This means that there is no need to reload the proxy service for the rules to take effect. + +Checkout Traefik' docs about `dynamic configuration `_ +and how to provide dynamic configuration through +`multiple separated files `_. diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index fd8946c..390dd6a 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -9,7 +9,37 @@ import toml from tornado.httpclient import HTTPClient, HTTPRequest, HTTPClientError import pytest -from tljh.config import reload_component, set_config_value, CONFIG_FILE, CONFIG_DIR +from tljh.config import ( + reload_component, + set_config_value, + unset_config_value, + CONFIG_FILE, + CONFIG_DIR, + STATE_DIR, +) + + +def send_request(url, max_sleep, validate_cert=True, username=None, password=None): + resp = None + for i in range(max_sleep): + time.sleep(i) + try: + req = HTTPRequest( + url, + method="GET", + auth_username=username, + auth_password=password, + validate_cert=validate_cert, + follow_redirects=True, + max_redirects=15, + ) + resp = HTTPClient().fetch(req) + break + except Exception as e: + print(e) + pass + + return resp def test_manual_https(preserve_config): @@ -55,32 +85,49 @@ def test_manual_https(preserve_config): # verify that our certificate was loaded by traefik assert server_cert == file_cert - for i in range(10): - time.sleep(i) - # verify that we can still connect to the hub - try: - req = HTTPRequest( - "https://127.0.0.1/hub/api", method="GET", validate_cert=False - ) - resp = HTTPClient().fetch(req) - break - except Exception as e: - pass + # verify that we can still connect to the hub + resp = send_request( + url="https://127.0.0.1/hub/api", max_sleep=10, validate_cert=False + ) assert resp.code == 200 + # cleanup shutil.rmtree(ssl_dir) + set_config_value(CONFIG_FILE, "https.enabled", False) + + reload_component("proxy") def test_extra_traefik_config(): - extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") - os.makedirs(extra_config_dir, exist_ok=True) + extra_static_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") + os.makedirs(extra_static_config_dir, exist_ok=True) - extra_config = { + dynamic_config_dir = os.path.join(STATE_DIR, "rules") + os.makedirs(dynamic_config_dir, exist_ok=True) + + + extra_static_config = { "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}}, "api": {"dashboard": True, "entrypoint": "no_auth_api"}, } + extra_dynamic_config = { + "frontends": { + "test": { + "backend": "test", + "routes": { + "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"} + }, + } + }, + "backends": { + # redirect to hub + "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}} + }, + } + + success = False for i in range(5): time.sleep(i) @@ -96,23 +143,31 @@ def test_extra_traefik_config(): assert success == True - # Load the extra config - with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file: - toml.dump(extra_config, extra_config_file) + # write the extra static config + with open( + os.path.join(extra_static_config_dir, "extra.toml"), "w+" + ) as extra_config_file: + toml.dump(extra_static_config, extra_config_file) + + # write the extra dynamic config + with open( + os.path.join(dynamic_config_dir, "extra_rules.toml"), "w+" + ) as extra_config_file: + toml.dump(extra_dynamic_config, extra_config_file) + + # load the extra config reload_component("proxy") - for i in range(5): - time.sleep(i) - try: - # The new dashboard entrypoint shouldn't require authentication anymore - req = HTTPRequest("http://127.0.0.1:9999/dashboard/", method="GET") - resp = HTTPClient().fetch(req) - break - except ConnectionRefusedError: - pass - # If the request didn't get through after 5 tries, this should fail + # the new dashboard entrypoint shouldn't require authentication anymore + resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5) assert resp.code == 200 + # test extra dynamic config + resp = send_request(url="http://127.0.0.1/the/hub/runs/here/too", max_sleep=5) + assert resp.code == 200 + assert resp.effective_url == "http://127.0.0.1/hub/login" + # cleanup - os.remove(os.path.join(extra_config_dir, "extra.toml")) - reload_component("proxy") + os.remove(os.path.join(extra_static_config_dir, "extra.toml")) + os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml")) + open(os.path.join(STATE_DIR, "traefik.toml"), "w").close() diff --git a/tljh/jupyterhub_config.py b/tljh/jupyterhub_config.py index 38fbaa9..e31e6e2 100644 --- a/tljh/jupyterhub_config.py +++ b/tljh/jupyterhub_config.py @@ -53,7 +53,7 @@ c.JupyterHub.hub_port = 15001 c.TraefikTomlProxy.should_start = False -dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules.toml') +dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules', 'rules.toml') c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path c.JupyterHub.proxy_class = TraefikTomlProxy diff --git a/tljh/systemd-units/traefik.service b/tljh/systemd-units/traefik.service index 5884fa4..1a27a5c 100644 --- a/tljh/systemd-units/traefik.service +++ b/tljh/systemd-units/traefik.service @@ -14,7 +14,7 @@ PrivateTmp=yes PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes -ReadWritePaths={install_prefix}/state/rules.toml +ReadWritePaths={install_prefix}/state/rules ReadWritePaths={install_prefix}/state/acme.json WorkingDirectory={install_prefix}/state ExecStart={install_prefix}/hub/bin/traefik \ diff --git a/tljh/traefik.py b/tljh/traefik.py index 2cbb312..997cf00 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -92,6 +92,7 @@ def ensure_traefik_config(state_dir): """Render the traefik.toml config file""" traefik_std_config_file = os.path.join(state_dir, "traefik.toml") traefik_extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") + traefik_dynamic_config_dir = os.path.join(state_dir, "rules") config = load_config() config['traefik_api']['basic_auth'] = compute_basic_auth( @@ -116,9 +117,12 @@ def ensure_traefik_config(state_dir): ): raise ValueError("Both email and domains must be set for letsencrypt") - # Ensure extra config dir exists and is private + # Ensure traefik extra static config dir exists and is private os.makedirs(traefik_extra_config_dir, mode=0o700, exist_ok=True) + # Ensure traefik dynamic config dir exists and is private + os.makedirs(traefik_dynamic_config_dir, mode=0o700, exist_ok=True) + try: # Load standard config file merge it with the extra config files into a dict extra_config = load_extra_config(traefik_extra_config_dir) @@ -131,7 +135,7 @@ def ensure_traefik_config(state_dir): os.fchmod(f.fileno(), 0o600) toml.dump(new_toml, f) - with open(os.path.join(state_dir, "rules.toml"), "w") as f: + with open(os.path.join(traefik_dynamic_config_dir, "rules.toml"), "w") as f: os.fchmod(f.fileno(), 0o600) # ensure acme.json exists and is private diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index 1b6de8f..4364c16 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -70,5 +70,5 @@ entryPoint = "https" {% endif %} [file] -filename = "rules.toml" +directory = "rules" watch = true