diff --git a/dev-requirements.txt b/dev-requirements.txt index 9902b41..0c50f19 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,4 +2,3 @@ pytest pytest-cov pytest-mock codecov -chardet==3.0.4 diff --git a/docs/howto/auth/dummy.rst b/docs/howto/auth/dummy.rst index 49f736b..5a92d89 100644 --- a/docs/howto/auth/dummy.rst +++ b/docs/howto/auth/dummy.rst @@ -26,7 +26,7 @@ Enabling the authenticator .. code-block:: bash - sudo tljh-config set auth.type dummyauthenticator.DummyAuthenticator + sudo tljh-config set auth.type dummy .. code-block:: bash diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index b89e17e..687d4a9 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -34,7 +34,7 @@ async def test_user_code_execute(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() async with User(username, hub_url, partial(login_dummy, password='')) as u: @@ -58,7 +58,7 @@ async def test_user_server_started_with_custom_base_url(): hub_url = f"http://localhost{base_url}" username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -81,7 +81,7 @@ async def test_user_admin_add(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -111,7 +111,7 @@ async def test_user_admin_remove(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -147,7 +147,7 @@ async def test_long_username(): hub_url = 'http://localhost' username = secrets.token_hex(32) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() try: @@ -183,7 +183,7 @@ async def test_user_group_adding(): # Create the group we want to add the user to system('groupadd somegroup') - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait() assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() @@ -223,7 +223,7 @@ async def test_idle_server_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() # Check every 10s for idle servers to cull assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() # Apart from servers, also cull users @@ -269,7 +269,7 @@ async def test_active_server_not_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait() + assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() # Check every 10s for idle servers to cull assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() # Apart from servers, also cull users diff --git a/setup.py b/setup.py index 00d7402..9c1e05b 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,9 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'ruamel.yaml==0.15.*', + 'ruamel.yaml==0.17.*', 'jinja2', - 'pluggy>0.7<1.0', + 'pluggy==1.*', 'passlib', 'backoff', 'requests', diff --git a/tests/test_configurer.py b/tests/test_configurer.py index d860069..98fe60d 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -98,13 +98,13 @@ def test_auth_dummy(): """ c = apply_mock_config({ 'auth': { - 'type': 'dummyauthenticator.DummyAuthenticator', + 'type': 'dummy', 'DummyAuthenticator': { 'password': 'test' } } }) - assert c.JupyterHub.authenticator_class == 'dummyauthenticator.DummyAuthenticator' + assert c.JupyterHub.authenticator_class == 'dummy' assert c.DummyAuthenticator.password == 'test' diff --git a/tljh/conda.py b/tljh/conda.py index 4b735e6..6827c47 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -96,6 +96,9 @@ def install_miniconda(installer_path, prefix): def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. + + Note that conda seem to update dependencies by default, so there is probably + no need to have a update parameter exposed for this function. """ conda_executable = [os.path.join(prefix, 'bin', 'mamba')] abspath = os.path.abspath(prefix) @@ -124,21 +127,20 @@ def ensure_conda_packages(prefix, packages): fix_permissions(prefix) -def ensure_pip_packages(prefix, packages): +def ensure_pip_packages(prefix, packages, upgrade=False): """ Ensure pip packages are installed in the given conda prefix. """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - - utils.run_subprocess(pip_executable + [ - 'install', - '--no-cache-dir', - ] + packages) + pip_cmd = pip_executable + ['install'] + if upgrade: + pip_cmd.append('--upgrade') + utils.run_subprocess(pip_cmd + packages) fix_permissions(prefix) -def ensure_pip_requirements(prefix, requirements_path): +def ensure_pip_requirements(prefix, requirements_path, upgrade=False): """ Ensure pip packages from given requirements_path are installed in given conda prefix. @@ -146,10 +148,8 @@ def ensure_pip_requirements(prefix, requirements_path): """ abspath = os.path.abspath(prefix) pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip'] - - utils.run_subprocess(pip_executable + [ - 'install', - '-r', - requirements_path - ]) + pip_cmd = pip_executable + ['install'] + if upgrade: + pip_cmd.append('--upgrade') + utils.run_subprocess(pip_cmd + ['--requirement', requirements_path]) fix_permissions(prefix) diff --git a/tljh/installer.py b/tljh/installer.py index 05b4ce4..0f531e6 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -124,25 +124,22 @@ def ensure_jupyterhub_package(prefix): 'libcurl4-openssl-dev', 'build-essential' ]) - conda.ensure_pip_packages(prefix, [ - 'pycurl==7.43.*' - ]) + conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True) conda.ensure_pip_packages( prefix, [ - "jupyterhub==1.4.0", - "jupyterhub-dummyauthenticator==0.3.1", - "jupyterhub-systemdspawner==0.15", - "jupyterhub-firstuseauthenticator==0.14.1", - "jupyterhub-nativeauthenticator==0.0.7", - "jupyterhub-ldapauthenticator==1.3.0", - "jupyterhub-tmpauthenticator==0.6", - "oauthenticator==0.10.0", - "jupyterhub-idle-culler==1.0", - "chardet==3.0.4", + "jupyterhub==1.*", + "jupyterhub-systemdspawner==0.15.*", + "jupyterhub-firstuseauthenticator==0.14.*", + "jupyterhub-nativeauthenticator==1.*", + "jupyterhub-ldapauthenticator==1.*", + "jupyterhub-tmpauthenticator==0.6.*", + "oauthenticator==14.*", + "jupyterhub-idle-culler==1.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba" ], + upgrade=True, ) traefik.ensure_traefik_binary(prefix) @@ -195,20 +192,28 @@ def ensure_user_environment(user_requirements_txt_file): conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda_version = '4.10.3' - conda.ensure_conda_packages(USER_ENV_PREFIX, [ - # Conda's latest version is on conda much more so than on PyPI. - 'conda==' + conda_version, - 'mamba==' + mambaforge_mamba_version, - ]) + conda.ensure_conda_packages( + USER_ENV_PREFIX, + [ + # Conda's latest version is on conda much more so than on PyPI. + 'conda==' + conda_version, + 'mamba==' + mambaforge_mamba_version, + ], + ) conda.ensure_pip_requirements( USER_ENV_PREFIX, os.path.join(HERE, 'requirements-base.txt'), + upgrade=True, ) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer - conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file) + conda.ensure_pip_requirements( + USER_ENV_PREFIX, + user_requirements_txt_file, + upgrade=True, + ) def ensure_admins(admin_password_list): @@ -315,7 +320,7 @@ def setup_plugins(plugins=None): """ # Install plugins if plugins: - conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) + conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins, upgrade=True) # Set up plugin infrastructure pm = pluggy.PluginManager('tljh') @@ -344,7 +349,11 @@ def run_plugin_actions(plugin_manager): logger.info('Installing {} hub pip packages collected from plugins: {}'.format( len(hub_pip_packages), ' '.join(hub_pip_packages) )) - conda.ensure_pip_packages(HUB_ENV_PREFIX, hub_pip_packages) + conda.ensure_pip_packages( + HUB_ENV_PREFIX, + hub_pip_packages, + upgrade=True, + ) # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) @@ -360,7 +369,11 @@ def run_plugin_actions(plugin_manager): logger.info('Installing {} user pip packages collected from plugins: {}'.format( len(user_pip_packages), ' '.join(user_pip_packages) )) - conda.ensure_pip_packages(USER_ENV_PREFIX, user_pip_packages) + conda.ensure_pip_packages( + USER_ENV_PREFIX, + user_pip_packages, + upgrade=True, + ) # Custom post install actions hook.tljh_post_install() diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 56768de..34e3d7a 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -1,18 +1,21 @@ +# When tljh.installer runs, the users' environment as typically found in +# /opt/tljh/user, is setup with these packages. +# # FIXME: a frozen version of this file should be used # pinning only direct dependencies is a recipe for broken environments! # JupyterHub + notebook package are base requirements for user environment -jupyterhub==1.4.0 -notebook==6.4.1 +jupyterhub==1.* +notebook==6.* # Install additional notebook frontends! jupyterlab==3.* -nteract-on-jupyter==2.1.* +nteract-on-jupyter==2.* # Install jupyterlab extensions from PyPI # nbgitpuller for easily pulling in Git repositories nbgitpuller==1.* # jupyter-resource-usage to show people how much RAM they are using -jupyter-resource-usage==0.5.* +jupyter-resource-usage==0.6.* # Most people consider ipywidgets to be part of the core notebook experience -ipywidgets==7.6.* +ipywidgets==7.* # Pin tornado tornado>=6.1