From 12316c32567969287338aaa268a78fcbfd887cc3 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 10 Jan 2023 16:40:59 -0800 Subject: [PATCH 1/3] Upgrade some packages Should upgrade base python version separately --- tljh/installer.py | 9 ++++----- tljh/requirements-base.txt | 15 ++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index 7e9948d..39af98c 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -121,14 +121,13 @@ def ensure_jupyterhub_package(prefix): conda.ensure_pip_packages( prefix, [ - "SQLAlchemy<2.0.0", - "jupyterhub==1.*", - "jupyterhub-systemdspawner==0.16.*", + "jupyterhub==3.1.*", + "jupyterhub-systemdspawner==0.17.*", "jupyterhub-firstuseauthenticator==1.*", "jupyterhub-nativeauthenticator==1.*", "jupyterhub-ldapauthenticator==1.*", - "jupyterhub-tmpauthenticator==0.6.*", - "oauthenticator==14.*", + "jupyterhub-tmpauthenticator==0.6", + "oauthenticator==15.*", "jupyterhub-idle-culler==1.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", ], diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 1985db7..7ac5d12 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -8,17 +8,14 @@ # For JupyterHub 1.x SQLAlchemy below 2.0.0 SQLAlchemy<2.0.0 # JupyterHub + notebook package are base requirements for user environment -jupyterhub==1.* -notebook==6.* +jupyterhub==3.1.* +notebook==6.5.* # Install additional notebook frontends! jupyterlab==3.* -nteract-on-jupyter==2.* -# Install jupyterlab extensions from PyPI +nteract-on-jupyter==2.1.* # nbgitpuller for easily pulling in Git repositories -nbgitpuller==1.* +nbgitpuller==1.1.* # jupyter-resource-usage to show people how much RAM they are using -jupyter-resource-usage==0.6.* +jupyter-resource-usage==0.7.* # Most people consider ipywidgets to be part of the core notebook experience -ipywidgets==7.* -# Pin tornado -tornado>=6.1 +ipywidgets==8.* From de1fc86b5ea57f3e214f35ec8c02acd4f0e60bbf Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 21 Mar 2023 10:29:56 +0100 Subject: [PATCH 2/3] only pin major versions in requirements-base.txt, installer.py --- tljh/installer.py | 4 ++-- tljh/requirements-base.txt | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index 39af98c..ac49086 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -121,12 +121,12 @@ def ensure_jupyterhub_package(prefix): conda.ensure_pip_packages( prefix, [ - "jupyterhub==3.1.*", + "jupyterhub==3.*", "jupyterhub-systemdspawner==0.17.*", "jupyterhub-firstuseauthenticator==1.*", "jupyterhub-nativeauthenticator==1.*", "jupyterhub-ldapauthenticator==1.*", - "jupyterhub-tmpauthenticator==0.6", + "jupyterhub-tmpauthenticator==0.6.*", "oauthenticator==15.*", "jupyterhub-idle-culler==1.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 7ac5d12..ba9ff85 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -5,16 +5,14 @@ # the requirements-txt-fixer pre-commit hook that sorted them and made # our integration tests fail. # -# For JupyterHub 1.x SQLAlchemy below 2.0.0 -SQLAlchemy<2.0.0 # JupyterHub + notebook package are base requirements for user environment -jupyterhub==3.1.* -notebook==6.5.* +jupyterhub==3.* +notebook==6.* # Install additional notebook frontends! jupyterlab==3.* -nteract-on-jupyter==2.1.* +nteract-on-jupyter==2.* # nbgitpuller for easily pulling in Git repositories -nbgitpuller==1.1.* +nbgitpuller==1.* # jupyter-resource-usage to show people how much RAM they are using jupyter-resource-usage==0.7.* # Most people consider ipywidgets to be part of the core notebook experience From 2d1c584ecac962e288e4a3b9e7a5bcbdc005c9b4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 28 Mar 2023 15:06:04 +0200 Subject: [PATCH 3/3] Fix wait/fail conditions in hub culling tests - actually check for running server, matching comments, not just user existence - catch asyncio.TimeoutError, not TimeoutError - fail if condition is not met, rather than passing in both cases - update some comments for accuracy (max age and timeout aren't the same) --- integration-tests/test_hub.py | 86 ++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index a056f62..fa5415d 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -346,12 +346,12 @@ async def test_idle_server_culled(): ) ).wait() ) - # Check every 10s for idle servers to cull + # Check every 5s for idle servers to cull assert ( 0 == await ( await asyncio.create_subprocess_exec( - *TLJH_CONFIG_PATH, "set", "services.cull.every", "10" + *TLJH_CONFIG_PATH, "set", "services.cull.every", "5" ) ).wait() ) @@ -364,12 +364,12 @@ async def test_idle_server_culled(): ) ).wait() ) - # Cull servers and users after 60s of activity + # Cull servers and users after 30s, regardless of activity assert ( 0 == await ( await asyncio.create_subprocess_exec( - *TLJH_CONFIG_PATH, "set", "services.cull.max_age", "60" + *TLJH_CONFIG_PATH, "set", "services.cull.max_age", "30" ) ).wait() ) @@ -388,25 +388,50 @@ async def test_idle_server_culled(): assert pwd.getpwnam(f"jupyter-{username}") is not None # Check that we can get to the user's server - r = await u.session.get( - u.hub_url / "hub/api/users" / username, - headers={"Referer": str(u.hub_url / "hub/")}, - ) + user_url = u.notebook_url / "api/status" + r = await u.session.get(user_url, allow_redirects=False) assert r.status == 200 - async def _check_culling_done(): - # Check that after 60s, the user and server have been culled and are not reacheable anymore + # Check that we can talk to JupyterHub itself + # use this as a proxy for whether the user still exists + async def hub_api_request(): r = await u.session.get( - u.hub_url / "hub/api/users" / username, + u.hub_url / "hub/api/user", headers={"Referer": str(u.hub_url / "hub/")}, + allow_redirects=False, ) - print(r.status) + return r + + r = await hub_api_request() + assert r.status == 200 + + # Wait for culling + # step 1: check if the server is still running + timeout = 100 + + async def server_stopped(): + """Has the server been stopped?""" + r = await u.session.get(user_url, allow_redirects=False) + print(f"{r.status} {r.url}") + return r.status != 200 + + await exponential_backoff( + server_stopped, + "Server still running!", + timeout=timeout, + ) + + # step 2. wait for user to be deleted + async def user_removed(): + # Check that after 60s, the user has been culled + r = await hub_api_request() + print(f"{r.status} {r.url}") return r.status == 403 await exponential_backoff( - _check_culling_done, - "Server culling failed!", - timeout=100, + user_removed, + "User still exists!", + timeout=timeout, ) @@ -429,12 +454,12 @@ async def test_active_server_not_culled(): ) ).wait() ) - # Check every 10s for idle servers to cull + # Check every 5s for idle servers to cull assert ( 0 == await ( await asyncio.create_subprocess_exec( - *TLJH_CONFIG_PATH, "set", "services.cull.every", "10" + *TLJH_CONFIG_PATH, "set", "services.cull.every", "5" ) ).wait() ) @@ -447,7 +472,7 @@ async def test_active_server_not_culled(): ) ).wait() ) - # Cull servers and users after 60s of activity + # Cull servers and users after 30s, regardless of activity assert ( 0 == await ( @@ -471,27 +496,24 @@ async def test_active_server_not_culled(): assert pwd.getpwnam(f"jupyter-{username}") is not None # Check that we can get to the user's server - r = await u.session.get( - u.hub_url / "hub/api/users" / username, - headers={"Referer": str(u.hub_url / "hub/")}, - ) + user_url = u.notebook_url / "api/status" + r = await u.session.get(user_url, allow_redirects=False) assert r.status == 200 - async def _check_culling_done(): + async def server_has_stopped(): # Check that after 30s, we can still reach the user's server - r = await u.session.get( - u.hub_url / "hub/api/users" / username, - headers={"Referer": str(u.hub_url / "hub/")}, - ) - print(r.status) + r = await u.session.get(user_url, allow_redirects=False) + print(f"{r.status} {r.url}") return r.status != 200 try: await exponential_backoff( - _check_culling_done, - "User's server is still reacheable!", + server_has_stopped, + "User's server is still reachable (good!)", timeout=30, ) - except TimeoutError: - # During the 30s timeout the user's server wasn't culled, which is what we intended. + except asyncio.TimeoutError: + # timeout error means the test passed - the server didn't go away while we were waiting pass + else: + pytest.fail(f"Server at {user_url} got culled prematurely!")