From 990135e17978403d59c5a5bbb7aacda89149c46b Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 19 Oct 2021 11:56:16 +0200 Subject: [PATCH 1/6] hub env: bump to require latest major version, less pinned now --- docs/howto/auth/dummy.rst | 2 +- integration-tests/test_hub.py | 16 ++++++++-------- tests/test_configurer.py | 4 ++-- tljh/installer.py | 21 ++++++++++----------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/howto/auth/dummy.rst b/docs/howto/auth/dummy.rst index 49f736b..5a92d89 100644 --- a/docs/howto/auth/dummy.rst +++ b/docs/howto/auth/dummy.rst @@ -26,7 +26,7 @@ Enabling the authenticator .. code-block:: bash - sudo tljh-config set auth.type dummyauthenticator.DummyAuthenticator + sudo tljh-config set auth.type dummy .. code-block:: bash diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index b89e17e..687d4a9 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -34,7 +34,7 @@ async def test_user_code_execute(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: @@ -58,7 +58,7 @@ async def test_user_server_started_with_custom_base_url(): hub_url = f"http://localhost{base_url}" username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -81,7 +81,7 @@ async def test_user_admin_add(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -111,7 +111,7 @@ async def test_user_admin_remove(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -147,7 +147,7 @@ async def test_long_username(): hub_url = 'http://localhost' username = secrets.token_hex(32) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() try: @@ -183,7 +183,7 @@ async def test_user_group_adding(): # Create the group we want to add the user to system('groupadd somegroup') - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -223,7 +223,7 @@ async def test_idle_server_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() # Check every 10s for idle servers to cull assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() # Apart from servers, also cull users @@ -269,7 +269,7 @@ async def test_active_server_not_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() # Check every 10s for idle servers to cull assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() # Apart from servers, also cull users diff --git a/tests/test_configurer.py b/tests/test_configurer.py index 1e57607..6a1637e 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -148,13 +148,13 @@ def test_auth_dummy(): """ c = apply_mock_config({ 'auth': { - 'type': 'dummyauthenticator.DummyAuthenticator', + 'type': 'dummy', 'DummyAuthenticator': { 'password': 'test' } } }) - assert c.JupyterHub.authenticator_class == 'dummyauthenticator.DummyAuthenticator' + assert c.JupyterHub.authenticator_class == 'dummy' assert c.DummyAuthenticator.password == 'test' diff --git a/tljh/installer.py b/tljh/installer.py index 05b4ce4..05b0ef7 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -125,22 +125,21 @@ def ensure_jupyterhub_package(prefix): 'build-essential' ]) conda.ensure_pip_packages(prefix, [ - 'pycurl==7.43.*' + 'pycurl==7.*' ]) conda.ensure_pip_packages( prefix, [ - "jupyterhub==1.4.0", - "jupyterhub-dummyauthenticator==0.3.1", - "jupyterhub-systemdspawner==0.15", - "jupyterhub-firstuseauthenticator==0.14.1", - "jupyterhub-nativeauthenticator==0.0.7", - "jupyterhub-ldapauthenticator==1.3.0", - "jupyterhub-tmpauthenticator==0.6", - "oauthenticator==0.10.0", - "jupyterhub-idle-culler==1.0", - "chardet==3.0.4", + "jupyterhub==1.*", + "jupyterhub-systemdspawner==0.15.*", + "jupyterhub-firstuseauthenticator==0.14.*", + "jupyterhub-nativeauthenticator==1.*", + "jupyterhub-ldapauthenticator==1.*", + "jupyterhub-tmpauthenticator==0.6.*", + "oauthenticator==14.*", + "jupyterhub-idle-culler==1.*", + "chardet==4.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba" ], ) From 8910ffb1ab50defcb1b05dcf9ea676a44a5f569a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 19 Oct 2021 11:57:22 +0200 Subject: [PATCH 2/6] user env: bump to require latest major version, less pinned now --- tljh/requirements-base.txt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 56768de..34e3d7a 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -1,18 +1,21 @@ +# When tljh.installer runs, the users' environment as typically found in +# /opt/tljh/user, is setup with these packages. +# # FIXME: a frozen version of this file should be used # pinning only direct dependencies is a recipe for broken environments! # JupyterHub + notebook package are base requirements for user environment -jupyterhub==1.4.0 -notebook==6.4.1 +jupyterhub==1.* +notebook==6.* # Install additional notebook frontends! jupyterlab==3.* -nteract-on-jupyter==2.1.* +nteract-on-jupyter==2.* # Install jupyterlab extensions from PyPI # nbgitpuller for easily pulling in Git repositories nbgitpuller==1.* # jupyter-resource-usage to show people how much RAM they are using -jupyter-resource-usage==0.5.* +jupyter-resource-usage==0.6.* # Most people consider ipywidgets to be part of the core notebook experience -ipywidgets==7.6.* +ipywidgets==7.* # Pin tornado tornado>=6.1 From 78bdf76860af78f65159ff10b3d781f9d8c162fb Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 19 Oct 2021 11:57:49 +0200 Subject: [PATCH 3/6] Update pinned tljh dependencies for: ruamel.yaml, pluggy --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 00d7402..9c1e05b 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,9 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'ruamel.yaml==0.15.*', + 'ruamel.yaml==0.17.*', 'jinja2', - 'pluggy>0.7<1.0', + 'pluggy==1.*', 'passlib', 'backoff', 'requests', From 61977840459fbbbe5e9224eb61279d882da90637 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 20 Oct 2021 23:32:14 +0200 Subject: [PATCH 4/6] Try stop pinning chardet --- dev-requirements.txt | 1 - tljh/installer.py | 1 - 2 files changed, 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 9902b41..0c50f19 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,4 +2,3 @@ pytest pytest-cov pytest-mock codecov -chardet==3.0.4 diff --git a/tljh/installer.py b/tljh/installer.py index 05b0ef7..a07f06e 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -139,7 +139,6 @@ def ensure_jupyterhub_package(prefix): "jupyterhub-tmpauthenticator==0.6.*", "oauthenticator==14.*", "jupyterhub-idle-culler==1.*", - "chardet==4.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba" ], ) From 125e12ca5e2a009cf1eebf7f1aa75bd5e0f9bd79 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Oct 2021 02:29:04 +0200 Subject: [PATCH 5/6] Let pip upgrade packages Since we now longer pin versions to the patch version, we should make an install cause the packages upgrade within the version constraints rathern than just settle for the current version if it is already installed. --- tljh/conda.py | 26 +++++++++++++------------- tljh/installer.py | 39 +++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/tljh/conda.py b/tljh/conda.py index 4b735e6..8983786 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -96,6 +96,9 @@ def install_miniconda(installer_path, prefix): def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. + + Note that conda seem to update dependencies by default, so there is probably + no need to have a update parameter exposed for this function. """ conda_executable = [os.path.join(prefix, 'bin', 'mamba')] abspath = os.path.abspath(prefix) @@ -124,21 +127,20 @@ def ensure_conda_packages(prefix, packages): fix_permissions(prefix) -def ensure_pip_packages(prefix, packages): +def ensure_pip_packages(prefix, packages, upgrade=False): """ Ensure pip packages are installed in the given conda prefix. """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - - utils.run_subprocess(pip_executable + [ - 'install', - '--no-cache-dir', - ] + packages) + pip_cmd = pip_executable + ['install', '--no-cache-dir'] + if upgrade: + pip_cmd.append('--upgrade') + utils.run_subprocess(pip_cmd + packages) fix_permissions(prefix) -def ensure_pip_requirements(prefix, requirements_path): +def ensure_pip_requirements(prefix, requirements_path, upgrade=False): """ Ensure pip packages from given requirements_path are installed in given conda prefix. @@ -146,10 +148,8 @@ def ensure_pip_requirements(prefix, requirements_path): """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - - utils.run_subprocess(pip_executable + [ - 'install', - '-r', - requirements_path - ]) + pip_cmd = pip_executable + ['install', '--no-cache-dir'] + if upgrade: + pip_cmd.append('--upgrade') + utils.run_subprocess(pip_cmd + ['--requirement', requirements_path]) fix_permissions(prefix) diff --git a/tljh/installer.py b/tljh/installer.py index a07f06e..0f531e6 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -124,9 +124,7 @@ def ensure_jupyterhub_package(prefix): 'libcurl4-openssl-dev', 'build-essential' ]) - conda.ensure_pip_packages(prefix, [ - 'pycurl==7.*' - ]) + conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True) conda.ensure_pip_packages( prefix, @@ -141,6 +139,7 @@ def ensure_jupyterhub_package(prefix): "jupyterhub-idle-culler==1.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba" ], + upgrade=True, ) traefik.ensure_traefik_binary(prefix) @@ -193,20 +192,28 @@ def ensure_user_environment(user_requirements_txt_file): conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda_version = '4.10.3' - conda.ensure_conda_packages(USER_ENV_PREFIX, [ - # Conda's latest version is on conda much more so than on PyPI. - 'conda==' + conda_version, - 'mamba==' + mambaforge_mamba_version, - ]) + conda.ensure_conda_packages( + USER_ENV_PREFIX, + [ + # Conda's latest version is on conda much more so than on PyPI. + 'conda==' + conda_version, + 'mamba==' + mambaforge_mamba_version, + ], + ) conda.ensure_pip_requirements( USER_ENV_PREFIX, os.path.join(HERE, 'requirements-base.txt'), + upgrade=True, ) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer - conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file) + conda.ensure_pip_requirements( + USER_ENV_PREFIX, + user_requirements_txt_file, + upgrade=True, + ) def ensure_admins(admin_password_list): @@ -313,7 +320,7 @@ def setup_plugins(plugins=None): """ # Install plugins if plugins: - conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) + conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins, upgrade=True) # Set up plugin infrastructure pm = pluggy.PluginManager('tljh') @@ -342,7 +349,11 @@ def run_plugin_actions(plugin_manager): logger.info('Installing {} hub pip packages collected from plugins: {}'.format( len(hub_pip_packages), ' '.join(hub_pip_packages) )) - conda.ensure_pip_packages(HUB_ENV_PREFIX, hub_pip_packages) + conda.ensure_pip_packages( + HUB_ENV_PREFIX, + hub_pip_packages, + upgrade=True, + ) # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) @@ -358,7 +369,11 @@ def run_plugin_actions(plugin_manager): logger.info('Installing {} user pip packages collected from plugins: {}'.format( len(user_pip_packages), ' '.join(user_pip_packages) )) - conda.ensure_pip_packages(USER_ENV_PREFIX, user_pip_packages) + conda.ensure_pip_packages( + USER_ENV_PREFIX, + user_pip_packages, + upgrade=True, + ) # Custom post install actions hook.tljh_post_install() From 3de51ff637fdcfe3ee897e80f47ac70ea34d17c3 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 27 Oct 2021 15:34:44 +0200 Subject: [PATCH 6/6] Remove --no-cache-dir flag --- tljh/conda.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tljh/conda.py b/tljh/conda.py index 8983786..6827c47 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -133,7 +133,7 @@ def ensure_pip_packages(prefix, packages, upgrade=False): """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - pip_cmd = pip_executable + ['install', '--no-cache-dir'] + pip_cmd = pip_executable + ['install'] if upgrade: pip_cmd.append('--upgrade') utils.run_subprocess(pip_cmd + packages) @@ -148,7 +148,7 @@ def ensure_pip_requirements(prefix, requirements_path, upgrade=False): """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - pip_cmd = pip_executable + ['install', '--no-cache-dir'] + pip_cmd = pip_executable + ['install'] if upgrade: pip_cmd.append('--upgrade') utils.run_subprocess(pip_cmd + ['--requirement', requirements_path])