From 77dc6a0e272b3d3ef9c42159eaa624edf5d2bed9 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 31 May 2019 16:52:51 -0700 Subject: [PATCH 1/3] Add hook to install packages in hub environment Required when installing additional authenticators or spawners --- .../plugins/simplest/tljh_simplest.py | 5 +++++ integration-tests/test_simplest_plugin.py | 10 ++++++++-- tljh/hooks.py | 6 ++++++ tljh/installer.py | 20 +++++++++++++------ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index 0da9a2a..053b1fd 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -17,6 +17,11 @@ def tljh_extra_user_pip_packages(): 'django', ] +@hookimpl +def tljh_extra_hub_pip_packages(): + return [ + 'there', + ] @hookimpl def tljh_extra_apt_packages(): diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 29b4fab..81193dd 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -4,7 +4,7 @@ Test simplest plugin from ruamel.yaml import YAML import os import subprocess -from tljh.config import CONFIG_FILE, USER_ENV_PREFIX +from tljh.config import CONFIG_FILE, USER_ENV_PREFIX, HUB_ENV_PREFIX yaml = YAML(typ='rt') @@ -18,7 +18,7 @@ def test_apt_packages(): def test_pip_packages(): """ - Test extra user pip packages are installed + Test extra user & hub pip packages are installed """ subprocess.check_call([ f'{USER_ENV_PREFIX}/bin/python3', @@ -26,6 +26,12 @@ def test_pip_packages(): 'import django' ]) + subprocess.check_call([ + f'{HUB_ENV_PREFIX}/bin/python3', + '-c', + 'import there' + ]) + def test_conda_packages(): """ diff --git a/tljh/hooks.py b/tljh/hooks.py index 8cd6d56..ece2a40 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -22,6 +22,12 @@ def tljh_extra_user_pip_packages(): """ pass +@hookspec +def tljh_extra_hub_pip_packages(): + """ + Return list of extra pip packages to install in the hub environment. + """ + pass @hookspec def tljh_extra_apt_packages(): diff --git a/tljh/installer.py b/tljh/installer.py index 3447e06..3a4ea93 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -381,21 +381,29 @@ def run_plugin_actions(plugin_manager, plugins): )) apt.install_packages(apt_packages) + # Install hub pip packages + hub_pip_packages = list(set(itertools.chain(*hook.tljh_extra_hub_pip_packages()))) + if hub_pip_packages: + logger.info('Installing {} hub pip packages collected from plugins: {}'.format( + len(hub_pip_packages), ' '.join(hub_pip_packages) + )) + conda.ensure_pip_packages(HUB_ENV_PREFIX, hub_pip_packages) + # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) if conda_packages: - logger.info('Installing {} conda packages collected from plugins: {}'.format( + logger.info('Installing {} user conda packages collected from plugins: {}'.format( len(conda_packages), ' '.join(conda_packages) )) conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) # Install pip packages - pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) - if pip_packages: - logger.info('Installing {} pip packages collected from plugins: {}'.format( - len(pip_packages), ' '.join(pip_packages) + user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) + if user_pip_packages: + logger.info('Installing {} user pip packages collected from plugins: {}'.format( + len(user_pip_packages), ' '.join(user_pip_packages) )) - conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages) + conda.ensure_pip_packages(USER_ENV_PREFIX, user_pip_packages) def ensure_config_yaml(plugin_manager): From b584bd1b2a0c58b407099fa2b65bf97ae4bbbc4d Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 31 May 2019 23:35:08 -0700 Subject: [PATCH 2/3] Add hook for custom jupyterhub_config.py content This lets extensions directly control how JupyterHub is configured --- integration-tests/plugins/simplest/tljh_simplest.py | 6 +++++- integration-tests/test_simplest_plugin.py | 9 +++++++++ tljh/hooks.py | 9 +++++++++ tljh/jupyterhub_config.py | 5 ++++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index 053b1fd..4e03f3c 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -35,4 +35,8 @@ def tljh_config_post_install(config): # Put an arbitrary marker we can test for config['simplest_plugin'] = { 'present': True - } \ No newline at end of file + } + +@hookimpl +def tljh_custom_jupyterhub_config(c): + c.JupyterHub.authenticator_class = 'tmpauthenticator.TmpAuthenticator' \ No newline at end of file diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 81193dd..37d17c4 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -2,6 +2,7 @@ Test simplest plugin """ from ruamel.yaml import YAML +import requests import os import subprocess from tljh.config import CONFIG_FILE, USER_ENV_PREFIX, HUB_ENV_PREFIX @@ -52,3 +53,11 @@ def test_config_hook(): data = yaml.load(f) assert data['simplest_plugin']['present'] + +def test_jupyterhub_config_hook(): + """ + Test that tmpauthenticator is enabled by our custom config plugin + """ + resp = requests.get('http://localhost/hub/tmplogin', allow_redirects=False) + assert resp.status_code == 302 + assert resp.headers['Location'] == '/hub/spawn' diff --git a/tljh/hooks.py b/tljh/hooks.py index ece2a40..8ab8d13 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -38,6 +38,15 @@ def tljh_extra_apt_packages(): """ pass +@hookspec +def tljh_custom_jupyterhub_config(c): + """ + Provide custom traitlet based config to JupyterHub. + + Anything you can put in `jupyterhub_config.py` can + be here. + """ + pass @hookspec def tljh_config_post_install(config): diff --git a/tljh/jupyterhub_config.py b/tljh/jupyterhub_config.py index ff1a34a..2806032 100644 --- a/tljh/jupyterhub_config.py +++ b/tljh/jupyterhub_config.py @@ -6,7 +6,7 @@ from glob import glob import os from systemdspawner import SystemdSpawner -from tljh import configurer, user +from tljh import configurer, user, hooks from tljh.config import INSTALL_PREFIX, USER_ENV_PREFIX, CONFIG_DIR from tljh.normalize import generate_system_username from tljh.yaml import yaml @@ -57,6 +57,9 @@ c.SystemdSpawner.unit_name_template = 'jupyter-{USERNAME}' tljh_config = configurer.load_config() configurer.apply_config(tljh_config, c) +# Let TLJH hooks modify `c` if they want +hooks.tljh_custom_jupyterhub_config(c) + # Load arbitrary .py config files if they exist. # This is our escape hatch extra_configs = sorted(glob(os.path.join(CONFIG_DIR, 'jupyterhub_config.d', '*.py'))) From b7f72a9f9eddab81acf2414f6486148921216218 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sat, 1 Jun 2019 00:32:23 -0700 Subject: [PATCH 3/3] Set up pluggy before calling hooks in jupyter_config.py --- tljh/jupyterhub_config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tljh/jupyterhub_config.py b/tljh/jupyterhub_config.py index 2806032..7f11bfa 100644 --- a/tljh/jupyterhub_config.py +++ b/tljh/jupyterhub_config.py @@ -4,6 +4,7 @@ JupyterHub config for the littlest jupyterhub. from glob import glob import os +import pluggy from systemdspawner import SystemdSpawner from tljh import configurer, user, hooks @@ -58,7 +59,13 @@ tljh_config = configurer.load_config() configurer.apply_config(tljh_config, c) # Let TLJH hooks modify `c` if they want -hooks.tljh_custom_jupyterhub_config(c) + +# Set up plugin infrastructure +pm = pluggy.PluginManager('tljh') +pm.add_hookspecs(hooks) +pm.load_setuptools_entrypoints('tljh') +# Call our custom configuration plugin +pm.hook.tljh_custom_jupyterhub_config(c=c) # Load arbitrary .py config files if they exist. # This is our escape hatch