From c31b9d3dead2526e39de5df2cdbb0ef8f00052d5 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 18 Jun 2020 17:29:50 +0300 Subject: [PATCH 1/5] Allow extending traefik dynamic config --- integration-tests/test_proxy.py | 43 ++++++++++++++++++++++++++++-- tljh/jupyterhub_config.py | 2 +- tljh/systemd-units/traefik.service | 2 +- tljh/traefik.py | 2 +- tljh/traefik.toml.tpl | 2 +- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index fd8946c..ced51a3 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -9,7 +9,13 @@ import toml from tornado.httpclient import HTTPClient, HTTPRequest, HTTPClientError import pytest -from tljh.config import reload_component, set_config_value, CONFIG_FILE, CONFIG_DIR +from tljh.config import ( + reload_component, + set_config_value, + CONFIG_FILE, + CONFIG_DIR, + STATE_DIR, +) def test_manual_https(preserve_config): @@ -72,7 +78,7 @@ def test_manual_https(preserve_config): shutil.rmtree(ssl_dir) -def test_extra_traefik_config(): +def test_extra_traefik_static_config(): extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") os.makedirs(extra_config_dir, exist_ok=True) @@ -116,3 +122,36 @@ def test_extra_traefik_config(): # cleanup os.remove(os.path.join(extra_config_dir, "extra.toml")) reload_component("proxy") + + +def test_extra_traefik_dynamic_config(): + dynamic_config_dir = os.path.join(STATE_DIR, "rules") + os.makedirs(dynamic_config_dir, exist_ok=True) + + extra_config = { + "frontends": { + "test": { + "backend": "test", + "routes": {"rule1": {"rule": "Path: /test/proxy"}}, + } + }, + "backends": { + "test": {"servers": {"server1": {"url": "https://mybinder.org/"}}} + }, + } + + # Load the extra config + with open( + os.path.join(dynamic_config_dir, "extra_rules.toml"), "w+" + ) as extra_config_file: + toml.dump(extra_config, extra_config_file) + reload_component("proxy") + + req = HTTPRequest("http://127.0.0.1/test/", method="GET") + resp = HTTPClient().fetch(req) + print(resp) + assert resp.code == 200 + + # cleanup + # os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml")) + # reload_component("proxy") diff --git a/tljh/jupyterhub_config.py b/tljh/jupyterhub_config.py index 38fbaa9..e31e6e2 100644 --- a/tljh/jupyterhub_config.py +++ b/tljh/jupyterhub_config.py @@ -53,7 +53,7 @@ c.JupyterHub.hub_port = 15001 c.TraefikTomlProxy.should_start = False -dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules.toml') +dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules', 'rules.toml') c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path c.JupyterHub.proxy_class = TraefikTomlProxy diff --git a/tljh/systemd-units/traefik.service b/tljh/systemd-units/traefik.service index 5884fa4..1a27a5c 100644 --- a/tljh/systemd-units/traefik.service +++ b/tljh/systemd-units/traefik.service @@ -14,7 +14,7 @@ PrivateTmp=yes PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes -ReadWritePaths={install_prefix}/state/rules.toml +ReadWritePaths={install_prefix}/state/rules ReadWritePaths={install_prefix}/state/acme.json WorkingDirectory={install_prefix}/state ExecStart={install_prefix}/hub/bin/traefik \ diff --git a/tljh/traefik.py b/tljh/traefik.py index 2cbb312..1a3105a 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -131,7 +131,7 @@ def ensure_traefik_config(state_dir): os.fchmod(f.fileno(), 0o600) toml.dump(new_toml, f) - with open(os.path.join(state_dir, "rules.toml"), "w") as f: + with open(os.path.join(state_dir, 'rules', "rules.toml"), "w") as f: os.fchmod(f.fileno(), 0o600) # ensure acme.json exists and is private diff --git a/tljh/traefik.toml.tpl b/tljh/traefik.toml.tpl index 1b6de8f..4364c16 100644 --- a/tljh/traefik.toml.tpl +++ b/tljh/traefik.toml.tpl @@ -70,5 +70,5 @@ entryPoint = "https" {% endif %} [file] -filename = "rules.toml" +directory = "rules" watch = true From c9b920d677cee28be245eef4b2d80a5c6154a8a6 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 18 Jun 2020 17:44:12 +0300 Subject: [PATCH 2/5] Ensure dynamic config dir exists --- tljh/traefik.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tljh/traefik.py b/tljh/traefik.py index 1a3105a..997cf00 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -92,6 +92,7 @@ def ensure_traefik_config(state_dir): """Render the traefik.toml config file""" traefik_std_config_file = os.path.join(state_dir, "traefik.toml") traefik_extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") + traefik_dynamic_config_dir = os.path.join(state_dir, "rules") config = load_config() config['traefik_api']['basic_auth'] = compute_basic_auth( @@ -116,9 +117,12 @@ def ensure_traefik_config(state_dir): ): raise ValueError("Both email and domains must be set for letsencrypt") - # Ensure extra config dir exists and is private + # Ensure traefik extra static config dir exists and is private os.makedirs(traefik_extra_config_dir, mode=0o700, exist_ok=True) + # Ensure traefik dynamic config dir exists and is private + os.makedirs(traefik_dynamic_config_dir, mode=0o700, exist_ok=True) + try: # Load standard config file merge it with the extra config files into a dict extra_config = load_extra_config(traefik_extra_config_dir) @@ -131,7 +135,7 @@ def ensure_traefik_config(state_dir): os.fchmod(f.fileno(), 0o600) toml.dump(new_toml, f) - with open(os.path.join(state_dir, 'rules', "rules.toml"), "w") as f: + with open(os.path.join(traefik_dynamic_config_dir, "rules.toml"), "w") as f: os.fchmod(f.fileno(), 0o600) # ensure acme.json exists and is private From b368b9716df1c45994deae1f07f0b581d885d328 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sat, 20 Jun 2020 02:05:53 +0300 Subject: [PATCH 3/5] Refactor proxy tests --- integration-tests/test_proxy.py | 98 ++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index ced51a3..f755470 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -12,12 +12,36 @@ import pytest from tljh.config import ( reload_component, set_config_value, + unset_config_value, CONFIG_FILE, CONFIG_DIR, STATE_DIR, ) +def send_request(url, max_sleep, validate_cert=True, username=None, password=None): + resp = None + for i in range(max_sleep): + time.sleep(i) + try: + req = HTTPRequest( + url, + method="GET", + auth_username=username, + auth_password=password, + validate_cert=validate_cert, + follow_redirects=True, + max_redirects=15, + ) + resp = HTTPClient().fetch(req) + break + except Exception as e: + print(e) + pass + + return resp + + def test_manual_https(preserve_config): ssl_dir = "/etc/tljh-ssl-test" key = ssl_dir + "/ssl.key" @@ -61,28 +85,25 @@ def test_manual_https(preserve_config): # verify that our certificate was loaded by traefik assert server_cert == file_cert - for i in range(10): - time.sleep(i) - # verify that we can still connect to the hub - try: - req = HTTPRequest( - "https://127.0.0.1/hub/api", method="GET", validate_cert=False - ) - resp = HTTPClient().fetch(req) - break - except Exception as e: - pass + # verify that we can still connect to the hub + resp = send_request( + url="https://127.0.0.1/hub/api", max_sleep=10, validate_cert=False + ) assert resp.code == 200 + # cleanup shutil.rmtree(ssl_dir) + set_config_value(CONFIG_FILE, "https.enabled", False) + + reload_component("proxy") def test_extra_traefik_static_config(): - extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") - os.makedirs(extra_config_dir, exist_ok=True) + extra_static_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") + os.makedirs(extra_static_config_dir, exist_ok=True) - extra_config = { + extra_static_config = { "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}}, "api": {"dashboard": True, "entrypoint": "no_auth_api"}, } @@ -102,25 +123,22 @@ def test_extra_traefik_static_config(): assert success == True - # Load the extra config - with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file: - toml.dump(extra_config, extra_config_file) + # write the extra static config + with open( + os.path.join(extra_static_config_dir, "extra.toml"), "w+" + ) as extra_config_file: + toml.dump(extra_static_config, extra_config_file) + + # load the extra config reload_component("proxy") - for i in range(5): - time.sleep(i) - try: - # The new dashboard entrypoint shouldn't require authentication anymore - req = HTTPRequest("http://127.0.0.1:9999/dashboard/", method="GET") - resp = HTTPClient().fetch(req) - break - except ConnectionRefusedError: - pass - # If the request didn't get through after 5 tries, this should fail + # the new dashboard entrypoint shouldn't require authentication anymore + resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5) assert resp.code == 200 # cleanup - os.remove(os.path.join(extra_config_dir, "extra.toml")) + os.remove(os.path.join(extra_static_config_dir, "extra.toml")) + open(os.path.join(STATE_DIR, "traefik.toml"), "w").close() reload_component("proxy") @@ -128,30 +146,34 @@ def test_extra_traefik_dynamic_config(): dynamic_config_dir = os.path.join(STATE_DIR, "rules") os.makedirs(dynamic_config_dir, exist_ok=True) - extra_config = { + extra_dynamic_config = { "frontends": { "test": { "backend": "test", - "routes": {"rule1": {"rule": "Path: /test/proxy"}}, + "routes": { + "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"} + }, } }, "backends": { - "test": {"servers": {"server1": {"url": "https://mybinder.org/"}}} + # redirect to hub + "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}} }, } - # Load the extra config + # write the extra dynamic config with open( os.path.join(dynamic_config_dir, "extra_rules.toml"), "w+" ) as extra_config_file: - toml.dump(extra_config, extra_config_file) + toml.dump(extra_dynamic_config, extra_config_file) + + # load the extra config reload_component("proxy") - req = HTTPRequest("http://127.0.0.1/test/", method="GET") - resp = HTTPClient().fetch(req) - print(resp) + # test extra dynamic config + resp = send_request(url="http://127.0.0.1/the/hub/runs/here/too", max_sleep=5) assert resp.code == 200 + assert resp.effective_url == "http://127.0.0.1/hub/login" # cleanup - # os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml")) - # reload_component("proxy") + os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml")) From 35275171654af2c642362da99aa184dd8ee8abcc Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sun, 21 Jun 2020 13:09:50 +0300 Subject: [PATCH 4/5] Add docs --- docs/topic/escape-hatch.rst | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/topic/escape-hatch.rst b/docs/topic/escape-hatch.rst index 9e2649f..d2fd869 100644 --- a/docs/topic/escape-hatch.rst +++ b/docs/topic/escape-hatch.rst @@ -8,11 +8,19 @@ Custom configuration snippets The two main TLJH components are **JupyterHub** and **Traefik**. * JupyterHub takes its configuration from the ``jupyterhub_config.py`` file. -* Traefik takes its configuration from the ``traefik.toml`` file. +* Traefik loads its: + * `static configuration `_ + from the ``traefik.toml`` file. + * `dynamic configuration `_ + from the ``rules`` directory. -These files are created by TLJH during installation and can be edited by the -user only through ``tljh-config``. Any direct modification to these files -is unsupported, and will cause hard to debug issues. +The ``jupyterhub_config.py`` and ``traefik.toml`` files are created by TLJH during installation +and can be edited by the user only through ``tljh-config``. The ``rules`` directory is also created +during install along with a ``rules/rules.toml`` file, to be used by JupyterHub to store the routing +table from users to their notebooks. + +.. note:: + Any direct modification to these files is unsupported, and will cause hard to debug issues. But because sometimes TLJH needs to be customized in ways that are not officially supported, an escape hatch has been introduced to allow easily extending the @@ -65,3 +73,22 @@ proxy for the new configuration to take effect: sudo tljh-config reload proxy .. warning:: This instructions might change when TLJH will switch to Traefik > 2.0 + +Extending ``rules.toml`` +======================== + +``Traefik`` is configured to load its routing table from the ``/opt/tljh/state/rules`` +directory. The existing ``rules.toml`` file inside this directory is used by +``jupyterhub-traefik-proxy`` to add the JupyterHub routes from users to their notebook servers +and shouldn't be modified. + +However, the routing table can be extended outside JupyterHub's scope using the ``rules`` +directory, by adding other dynamic configuration files with the desired routing rules. + +.. note:: + * Any files in ``/opt/tljh/state/rules`` that end in ``.toml`` will be hot reload by Traefik. + This means that there is no need to reload the proxy service for the rules to take effect. + +Checkout Traefik' docs about `dynamic configuration `_ +and how to provide dynamic configuration through +`multiple separated files `_. From 15974a359fdd8a6a21cb01cc56453436deb7c204 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sun, 21 Jun 2020 13:55:52 +0300 Subject: [PATCH 5/5] Group tests for fewer reloads --- integration-tests/test_proxy.py | 60 +++++++++++++++------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index f755470..390dd6a 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -99,15 +99,35 @@ def test_manual_https(preserve_config): reload_component("proxy") -def test_extra_traefik_static_config(): +def test_extra_traefik_config(): extra_static_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d") os.makedirs(extra_static_config_dir, exist_ok=True) + dynamic_config_dir = os.path.join(STATE_DIR, "rules") + os.makedirs(dynamic_config_dir, exist_ok=True) + + extra_static_config = { "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}}, "api": {"dashboard": True, "entrypoint": "no_auth_api"}, } + extra_dynamic_config = { + "frontends": { + "test": { + "backend": "test", + "routes": { + "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"} + }, + } + }, + "backends": { + # redirect to hub + "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}} + }, + } + + success = False for i in range(5): time.sleep(i) @@ -129,38 +149,6 @@ def test_extra_traefik_static_config(): ) as extra_config_file: toml.dump(extra_static_config, extra_config_file) - # load the extra config - reload_component("proxy") - - # the new dashboard entrypoint shouldn't require authentication anymore - resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5) - assert resp.code == 200 - - # cleanup - os.remove(os.path.join(extra_static_config_dir, "extra.toml")) - open(os.path.join(STATE_DIR, "traefik.toml"), "w").close() - reload_component("proxy") - - -def test_extra_traefik_dynamic_config(): - dynamic_config_dir = os.path.join(STATE_DIR, "rules") - os.makedirs(dynamic_config_dir, exist_ok=True) - - extra_dynamic_config = { - "frontends": { - "test": { - "backend": "test", - "routes": { - "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"} - }, - } - }, - "backends": { - # redirect to hub - "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}} - }, - } - # write the extra dynamic config with open( os.path.join(dynamic_config_dir, "extra_rules.toml"), "w+" @@ -170,10 +158,16 @@ def test_extra_traefik_dynamic_config(): # load the extra config reload_component("proxy") + # the new dashboard entrypoint shouldn't require authentication anymore + resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5) + assert resp.code == 200 + # test extra dynamic config resp = send_request(url="http://127.0.0.1/the/hub/runs/here/too", max_sleep=5) assert resp.code == 200 assert resp.effective_url == "http://127.0.0.1/hub/login" # cleanup + os.remove(os.path.join(extra_static_config_dir, "extra.toml")) os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml")) + open(os.path.join(STATE_DIR, "traefik.toml"), "w").close()