From e0d6d32a143539863716b6652a9053363e3ce226 Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Mon, 19 Oct 2020 18:54:47 +0200 Subject: [PATCH 1/9] Temporary modification of bootstrap git repo for testing --- bootstrap/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 5fcd7b1..dab566e 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -292,7 +292,7 @@ def main(): pip_flags.append('--editable') tljh_repo_path = os.environ.get( 'TLJH_BOOTSTRAP_PIP_SPEC', - 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git' + 'git+https://github.com/jeanmarcalkazzi/the-littlest-jupyterhub' ) # Upgrade pip From 66cd578d752e6405a11fe71c00af4738348a312f Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Mon, 19 Oct 2020 19:48:23 +0200 Subject: [PATCH 2/9] Adding update_base_url from tljh config --- tljh/config.py | 5 ++++- tljh/configurer.py | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index e24abce..a30f176 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -240,9 +240,12 @@ def remove_config_value(config_path, key_path, value): def check_hub_ready(): from .configurer import load_config + + base_url = load_config()['base_url'] + base_url = base_url[:-1] if base_url[-1] == '/' else base_url http_port = load_config()['http']['port'] try: - r = requests.get('http://127.0.0.1:%d/hub/api' % http_port, verify=False) + r = requests.get('http://127.0.0.1:%d%s/hub/api' % (http_port, base_url), verify=False) return r.status_code == 200 except: return False diff --git a/tljh/configurer.py b/tljh/configurer.py index 73f602b..018c7d2 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -17,6 +17,7 @@ from .yaml import yaml # Default configuration for tljh # User provided config is merged into this default = { + 'base_url': '/', 'auth': { 'type': 'firstuseauthenticator.FirstUseAuthenticator', 'FirstUseAuthenticator': { @@ -69,6 +70,7 @@ default = { } } + def load_config(config_file=CONFIG_FILE): """Load the current config as a dictionary @@ -92,6 +94,7 @@ def apply_config(config_overrides, c): """ tljh_config = _merge_dictionaries(dict(default), config_overrides) + update_base_url(c, tljh_config) update_auth(c, tljh_config) update_userlists(c, tljh_config) update_usergroups(c, tljh_config) @@ -115,7 +118,7 @@ def load_traefik_api_credentials(): proxy_secret_path = os.path.join(STATE_DIR, 'traefik-api.secret') if not os.path.exists(proxy_secret_path): return {} - with open(proxy_secret_path,'r') as f: + with open(proxy_secret_path, 'r') as f: password = f.read() return { 'traefik_api': { @@ -134,6 +137,13 @@ def load_secrets(): return config +def update_base_url(c, config): + """ + Update base_url of JupyterHub through tljh config + """ + c.JupyterHub.base_url = config['base_url'] + + def update_auth(c, config): """ Set auth related configuration from YAML config file @@ -218,7 +228,7 @@ def set_cull_idle_service(config): Set Idle Culler service """ cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler' + sys.executable, '-m', 'jupyterhub_idle_culler' ] cull_config = config['services']['cull'] print() From f7992069208c15b8ec968ee78a9edb7d5e0253fb Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Tue, 20 Oct 2020 00:14:15 +0200 Subject: [PATCH 3/9] Add relevant tests and formatting --- tests/test_configurer.py | 38 +++++++++++++++++++++++++++----------- tljh/config.py | 13 ++++++++++--- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/tests/test_configurer.py b/tests/test_configurer.py index fa3090e..5380861 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -2,6 +2,7 @@ Test configurer """ +from traitlets import Dict import os import sys @@ -62,6 +63,22 @@ def apply_mock_config(overrides): return c +def test_default_base_url(): + """ + Test default JupyterHub base_url + """ + c = apply_mock_config({}) + assert c.JupyterHub.base_url == '/' + + +def test_set_base_url(): + """ + Test set JupyterHub base_url + """ + c = apply_mock_config({'base_url': '/custom-base'}) + assert c.JupyterHub.base_url == '/custom-base' + + def test_default_memory_limit(): """ Test default per user memory limit @@ -129,7 +146,7 @@ def test_auth_dummy(): assert c.JupyterHub.authenticator_class == 'dummyauthenticator.DummyAuthenticator' assert c.DummyAuthenticator.password == 'test' -from traitlets import Dict + def test_user_groups(): """ Test setting user groups @@ -143,9 +160,9 @@ def test_user_groups(): } }) assert c.UserCreatingSpawner.user_groups == { - "g1": ["u1", "u2"], - "g2": ["u3", "u4"] - } + "g1": ["u1", "u2"], + "g2": ["u3", "u4"] + } def test_auth_firstuse(): @@ -213,9 +230,9 @@ def test_cull_service_default(): c = apply_mock_config({}) cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler', - '--timeout=600', '--cull-every=60', '--concurrency=5', - '--max-age=0' + sys.executable, '-m', 'jupyterhub_idle_culler', + '--timeout=600', '--cull-every=60', '--concurrency=5', + '--max-age=0' ] assert c.JupyterHub.services == [{ 'name': 'cull-idle', @@ -238,9 +255,9 @@ def test_set_cull_service(): } }) cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler', - '--timeout=600', '--cull-every=10', '--concurrency=5', - '--max-age=60', '--cull-users' + sys.executable, '-m', 'jupyterhub_idle_culler', + '--timeout=600', '--cull-every=10', '--concurrency=5', + '--max-age=60', '--cull-users' ] assert c.JupyterHub.services == [{ 'name': 'cull-idle', @@ -276,4 +293,3 @@ def test_auth_native(): }) assert c.JupyterHub.authenticator_class == 'nativeauthenticator.NativeAuthenticator' assert c.NativeAuthenticator.open_signup == True - diff --git a/tljh/config.py b/tljh/config.py index a30f176..ec235f4 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -61,6 +61,7 @@ def set_item_in_config(config, property_path, value): return config_copy + def unset_item_from_config(config, property_path): """ Unset key at property_path in config & return new config. @@ -105,6 +106,7 @@ def unset_item_from_config(config, property_path): return config_copy + def add_item_to_config(config, property_path, value): """ Add an item to a list in config. @@ -238,6 +240,7 @@ def remove_config_value(config_path, key_path, value): with open(config_path, 'w') as f: yaml.dump(config, f) + def check_hub_ready(): from .configurer import load_config @@ -250,6 +253,7 @@ def check_hub_ready(): except: return False + def reload_component(component): """ Reload a TLJH component. @@ -392,13 +396,16 @@ def main(argv=None): if args.action == 'show': show_config(args.config_path) elif args.action == 'set': - set_config_value(args.config_path, args.key_path, parse_value(args.value)) + set_config_value(args.config_path, args.key_path, + parse_value(args.value)) elif args.action == 'unset': unset_config_value(args.config_path, args.key_path) elif args.action == 'add-item': - add_config_value(args.config_path, args.key_path, parse_value(args.value)) + add_config_value(args.config_path, args.key_path, + parse_value(args.value)) elif args.action == 'remove-item': - remove_config_value(args.config_path, args.key_path, parse_value(args.value)) + remove_config_value(args.config_path, args.key_path, + parse_value(args.value)) elif args.action == 'reload': reload_component(args.component) else: From 42e06335f801fde775d86cf3920c6c1b4006b85b Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Tue, 20 Oct 2020 00:16:20 +0200 Subject: [PATCH 4/9] re-add original git url in bootstrap --- bootstrap/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index dab566e..5fcd7b1 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -292,7 +292,7 @@ def main(): pip_flags.append('--editable') tljh_repo_path = os.environ.get( 'TLJH_BOOTSTRAP_PIP_SPEC', - 'git+https://github.com/jeanmarcalkazzi/the-littlest-jupyterhub' + 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git' ) # Upgrade pip From e4df0c4a59b8d8eed48e0b267fc815805229a329 Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Tue, 20 Oct 2020 00:20:42 +0200 Subject: [PATCH 5/9] Add base_url config possibility in docs --- docs/topic/tljh-config.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/topic/tljh-config.rst b/docs/topic/tljh-config.rst index fe70fa1..6c44f53 100644 --- a/docs/topic/tljh-config.rst +++ b/docs/topic/tljh-config.rst @@ -63,6 +63,13 @@ file. If what you want is only to change the property's value, you should use Some of the existing ```` are listed below by categories: +.. _tljh-base_url: + +Base URL +-------------- + + Use ``base_url`` to determine the base URL used by JupyterHub. This parameter will + be passed straight to ``c.JupyterHub.base_url``. .. _tljh-set-auth: From b1db8928e28b466e124d6925ae30a8d1bbccd68f Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Tue, 20 Oct 2020 00:22:04 +0200 Subject: [PATCH 6/9] fix --- --- docs/topic/tljh-config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topic/tljh-config.rst b/docs/topic/tljh-config.rst index 6c44f53..a1a04db 100644 --- a/docs/topic/tljh-config.rst +++ b/docs/topic/tljh-config.rst @@ -66,7 +66,7 @@ Some of the existing ```` are listed below by categories: .. _tljh-base_url: Base URL --------------- +-------- Use ``base_url`` to determine the base URL used by JupyterHub. This parameter will be passed straight to ``c.JupyterHub.base_url``. From 2eed619745a583c33dfed681d3845c545d3b17ee Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Mon, 26 Oct 2020 21:22:11 +0100 Subject: [PATCH 7/9] Add integration test for user code execution with custom base_url --- integration-tests/test_hub.py | 199 ++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 83 deletions(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 22dfbbf..b57837e 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -18,6 +18,7 @@ from tljh.normalize import generate_system_username # This catches issues with PATH TLJH_CONFIG_PATH = ['sudo', 'tljh-config'] + def test_hub_up(): r = requests.get('http://127.0.0.1') r.raise_for_status() @@ -37,13 +38,41 @@ async def test_user_code_execute(): assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - await u.ensure_server_simulate() - await u.start_kernel() - await u.assert_code_output("5 * 4", "20", 5, 5) + await u.login() + await u.ensure_server_simulate() + await u.start_kernel() + await u.assert_code_output("5 * 4", "20", 5, 5) - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None + + +@pytest.mark.asyncio +async def test_user_code_execute_with_custom_base_url(): + """ + User logs in, starts a server with a custom base_url & executes code + """ + # This *must* be localhost, not an IP + # aiohttp throws away cookies if we are connecting to an IP! + hub_url = 'http://localhost' + username = secrets.token_hex(8) + + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', '/custom-base')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + + async with User(username, hub_url, partial(login_dummy, password='')) as u: + await u.login() + await u.ensure_server_simulate() + await u.start_kernel() + await u.assert_code_output("5 * 4", "20", 5, 5) + + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None + + # unset base_url to avoid problems with other tests + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'unset', 'base_url')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @pytest.mark.asyncio @@ -61,14 +90,15 @@ async def test_user_admin_add(): assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - await u.ensure_server_simulate() + await u.login() + await u.ensure_server_simulate() - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None - # Assert that the user has admin rights - assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem + # Assert that the user has admin rights + assert f'jupyter-{username}' in grp.getgrnam( + 'jupyterhub-admins').gr_mem # FIXME: Make this test pass @@ -90,23 +120,25 @@ async def test_user_admin_remove(): assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - await u.ensure_server_simulate() + await u.login() + await u.ensure_server_simulate() - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None - # Assert that the user has admin rights - assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem + # Assert that the user has admin rights + assert f'jupyter-{username}' in grp.getgrnam( + 'jupyterhub-admins').gr_mem - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() - await u.stop_server() - await u.ensure_server_simulate() + await u.stop_server() + await u.ensure_server_simulate() - # Assert that the user does *not* have admin rights - assert f'jupyter-{username}' not in grp.getgrnam('jupyterhub-admins').gr_mem + # Assert that the user does *not* have admin rights + assert f'jupyter-{username}' not in grp.getgrnam( + 'jupyterhub-admins').gr_mem @pytest.mark.asyncio @@ -124,14 +156,14 @@ async def test_long_username(): try: async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - await u.ensure_server_simulate() + await u.login() + await u.ensure_server_simulate() - # Assert that the user exists - system_username = generate_system_username(f'jupyter-{username}') - assert pwd.getpwnam(system_username) is not None + # Assert that the user exists + system_username = generate_system_username(f'jupyter-{username}') + assert pwd.getpwnam(system_username) is not None - await u.stop_server() + await u.stop_server() except: # If we have any errors, print jupyterhub logs before exiting subprocess.check_call([ @@ -161,19 +193,19 @@ async def test_user_group_adding(): try: async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - await u.ensure_server_simulate() + await u.login() + await u.ensure_server_simulate() - # Assert that the user exists - system_username = generate_system_username(f'jupyter-{username}') - assert pwd.getpwnam(system_username) is not None + # Assert that the user exists + system_username = generate_system_username(f'jupyter-{username}') + assert pwd.getpwnam(system_username) is not None - # Assert that the user was added to the specified group - assert f'jupyter-{username}' in grp.getgrnam('somegroup').gr_mem + # Assert that the user was added to the specified group + assert f'jupyter-{username}' in grp.getgrnam('somegroup').gr_mem - await u.stop_server() - # Delete the group - system('groupdel somegroup') + await u.stop_server() + # Delete the group + system('groupdel somegroup') except: # If we have any errors, print jupyterhub logs before exiting subprocess.check_call([ @@ -205,29 +237,30 @@ async def test_idle_server_culled(): assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - # Start user's server - await u.ensure_server_simulate() - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None + await u.login() + # Start user's server + await u.ensure_server_simulate() + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None - # Check that we can get to the user's server + # Check that we can get to the user's server + r = await u.session.get(u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}) + assert r.status == 200 + + async def _check_culling_done(): + # Check that after 60s, the user and server have been culled and are not reacheable anymore r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) - assert r.status == 200 + headers={'Referer': str(u.hub_url / 'hub/')}) + print(r.status) + return r.status == 403 - async def _check_culling_done(): - # Check that after 60s, the user and server have been culled and are not reacheable anymore - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) - print(r.status) - return r.status == 403 + await exponential_backoff( + _check_culling_done, + "Server culling failed!", + timeout=100, + ) - await exponential_backoff( - _check_culling_done, - "Server culling failed!", - timeout=100, - ) @pytest.mark.asyncio async def test_active_server_not_culled(): @@ -250,30 +283,30 @@ async def test_active_server_not_culled(): assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: - await u.login() - # Start user's server - await u.ensure_server_simulate() - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None + await u.login() + # Start user's server + await u.ensure_server_simulate() + # Assert that the user exists + assert pwd.getpwnam(f'jupyter-{username}') is not None - # Check that we can get to the user's server + # Check that we can get to the user's server + r = await u.session.get(u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}) + assert r.status == 200 + + async def _check_culling_done(): + # Check that after 30s, we can still reach the user's server r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) - assert r.status == 200 + headers={'Referer': str(u.hub_url / 'hub/')}) + print(r.status) + return r.status != 200 - async def _check_culling_done(): - # Check that after 30s, we can still reach the user's server - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) - print(r.status) - return r.status != 200 - - try: - await exponential_backoff( - _check_culling_done, - "User's server is still reacheable!", - timeout=30, - ) - except TimeoutError: - # During the 30s timeout the user's server wasn't culled, which is what we intended. - pass + try: + await exponential_backoff( + _check_culling_done, + "User's server is still reacheable!", + timeout=30, + ) + except TimeoutError: + # During the 30s timeout the user's server wasn't culled, which is what we intended. + pass From 0a0abf4ff26f94102c94ac1e2ef5ea175b1edc3f Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Mon, 26 Oct 2020 22:16:15 +0100 Subject: [PATCH 8/9] Add base --- integration-tests/test_hub.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index b57837e..5af2932 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -54,21 +54,17 @@ async def test_user_code_execute_with_custom_base_url(): """ # This *must* be localhost, not an IP # aiohttp throws away cookies if we are connecting to an IP! - hub_url = 'http://localhost' + base_url = "/custom-base" + hub_url = f"http://localhost{base_url}" username = secrets.token_hex(8) assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', '/custom-base')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() await u.ensure_server_simulate() - await u.start_kernel() - await u.assert_code_output("5 * 4", "20", 5, 5) - - # Assert that the user exists - assert pwd.getpwnam(f'jupyter-{username}') is not None # unset base_url to avoid problems with other tests assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'unset', 'base_url')).wait() From 70853059ff14f48c39e932461ff52bb18fbaa948 Mon Sep 17 00:00:00 2001 From: Jean-Marc Alkazzi Date: Mon, 26 Oct 2020 22:17:37 +0100 Subject: [PATCH 9/9] Test user server started with custom base_url --- integration-tests/test_hub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 5af2932..b89e17e 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -48,7 +48,7 @@ async def test_user_code_execute(): @pytest.mark.asyncio -async def test_user_code_execute_with_custom_base_url(): +async def test_user_server_started_with_custom_base_url(): """ User logs in, starts a server with a custom base_url & executes code """