From d12345e72a875e9654d96b71fb0ac4a35c0cdced Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sat, 11 Aug 2018 00:42:28 -0700 Subject: [PATCH 1/6] Add plugin support to the installer TLJH is a great base to build 'stacks' on top of for various use cases. These 'stacks' should be built by people who are domain experts in their fields, and easily updateable with new TLJH versions. Extension points need to be very clearly defined & evolvable, so we can modify TLJH without fear of breaking everything. [pluggy](https://pluggy.readthedocs.io/) is the plugin mechanism for pytest spun out into its own library, and fits our requirements well. There is an experimental pangeo stack in progress at https://github.com/yuvipanda/tljh-pangeo for an example of how this would work --- setup.py | 1 + tljh/hooks.py | 33 ++++++++++++++++++++++++++++++ tljh/installer.py | 52 ++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 tljh/hooks.py diff --git a/setup.py b/setup.py index f50745e..e01f4e1 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ setup( 'pyyaml==3.*', 'ruamel.yaml==0.15.*', 'jinja2', + 'pluggy>0.7<1.0' ], entry_points={ 'console_scripts': [ diff --git a/tljh/hooks.py b/tljh/hooks.py new file mode 100644 index 0000000..f1a673e --- /dev/null +++ b/tljh/hooks.py @@ -0,0 +1,33 @@ +""" +Hook specifications that pluggy plugins can override +""" +import pluggy + +hookspec = pluggy.HookspecMarker('tljh') +hookimpl = pluggy.HookimplMarker('tljh') + + +@hookspec +def tljh_extra_user_conda_packages(): + """ + Return list of extra conda packages to install in user environment. + """ + pass + + +@hookspec +def tljh_extra_user_pip_packages(): + """ + Return list of extra pip packages to install in user environment. + """ + pass + + +@hookspec +def tljh_extra_apt_packages(): + """ + Return list of extra apt packages to install in the user environment. + + These will be installed before additional pip or conda packages. + """ + pass \ No newline at end of file diff --git a/tljh/installer.py b/tljh/installer.py index fa2b838..c384338 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -2,15 +2,17 @@ import argparse import os import secrets import subprocess +import itertools import sys import time import logging from urllib.error import HTTPError from urllib.request import urlopen, URLError +import pluggy from ruamel.yaml import YAML -from tljh import conda, systemd, traefik, user, apt +from tljh import conda, systemd, traefik, user, apt, hooks from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR HERE = os.path.abspath(os.path.dirname(__file__)) @@ -305,6 +307,44 @@ def ensure_symlinks(prefix): return os.symlink(tljh_config_src, tljh_config_dest) +def run_plugin_actions(plugins): + """ + Run installer hooks defined in plugins + """ + + # Install plugins + if plugins: + conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) + + # Set up plugin infrastructure + pm = pluggy.PluginManager('tljh') + pm.add_hookspecs(hooks) + pm.load_setuptools_entrypoints('tljh') + + # Install apt packages + apt_packages = list(set(itertools.chain(*pm.hook.tljh_extra_apt_packages()))) + if apt_packages: + logger.info('Installing {} apt packages collected from plugins: {}'.format( + len(apt_packages), ' '.join(apt_packages) + )) + apt.install_packages(apt_packages) + + # Install conda packages + conda_packages = list(set(itertools.chain(*pm.hook.tljh_extra_user_conda_packages()))) + if conda_packages: + logger.info('Installing {} conda packages collected from plugins: {}'.format( + len(conda_packages), ' '.join(conda_packages) + )) + conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) + + # Install pip packages + pip_packages = list(set(itertools.chain(*pm.hook.tljh_extra_user_pip_packages()))) + if pip_packages: + logger.info('Installing {} pip packages collected from plugins: {}'.format( + len(pip_packages), ' '.join(pip_packages) + )) + conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages) + def main(): argparser = argparse.ArgumentParser() @@ -317,11 +357,14 @@ def main(): '--user-requirements-txt-url', help='URL to a requirements.txt file that should be installed in the user enviornment' ) + argparser.add_argument( + '--plugin', + nargs='*', + help='Plugin pip-specs to install' + ) args = argparser.parse_args() - - ensure_admins(args.admin) ensure_usergroups() @@ -335,6 +378,9 @@ def main(): ensure_jupyterhub_running() ensure_symlinks(HUB_ENV_PREFIX) + # Run installer plugins last + run_plugin_actions(args.plugin) + logger.info("Done!") From 841c25e1bdc74db2f264d11b6b0b35c9bf0ff347 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sat, 11 Aug 2018 01:19:51 -0700 Subject: [PATCH 2/6] Add plugin hook to modify config.yaml post install --- tljh/hooks.py | 13 +++++++++++++ tljh/installer.py | 45 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/tljh/hooks.py b/tljh/hooks.py index f1a673e..8cd6d56 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -30,4 +30,17 @@ def tljh_extra_apt_packages(): These will be installed before additional pip or conda packages. """ + pass + + +@hookspec +def tljh_config_post_install(config): + """ + Modify on-disk tljh-config after installation. + + config is a dict-like object that should be modified + in-place. The contents of the on-disk config.yaml will + be the serialized contents of config, so try to not + overwrite anything the user might have explicitly set. + """ pass \ No newline at end of file diff --git a/tljh/installer.py b/tljh/installer.py index c384338..301e83c 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -13,7 +13,7 @@ import pluggy from ruamel.yaml import YAML from tljh import conda, systemd, traefik, user, apt, hooks -from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR +from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR, CONFIG_FILE HERE = os.path.abspath(os.path.dirname(__file__)) @@ -307,11 +307,11 @@ def ensure_symlinks(prefix): return os.symlink(tljh_config_src, tljh_config_dest) -def run_plugin_actions(plugins): - """ - Run installer hooks defined in plugins - """ +def setup_plugins(plugins): + """ + Install plugins & setup a pluginmanager + """ # Install plugins if plugins: conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) @@ -321,8 +321,15 @@ def run_plugin_actions(plugins): pm.add_hookspecs(hooks) pm.load_setuptools_entrypoints('tljh') + return pm + +def run_plugin_actions(plugin_manager, plugins): + """ + Run installer hooks defined in plugins + """ + hook = plugin_manager.hook # Install apt packages - apt_packages = list(set(itertools.chain(*pm.hook.tljh_extra_apt_packages()))) + apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages()))) if apt_packages: logger.info('Installing {} apt packages collected from plugins: {}'.format( len(apt_packages), ' '.join(apt_packages) @@ -330,7 +337,7 @@ def run_plugin_actions(plugins): apt.install_packages(apt_packages) # Install conda packages - conda_packages = list(set(itertools.chain(*pm.hook.tljh_extra_user_conda_packages()))) + conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) if conda_packages: logger.info('Installing {} conda packages collected from plugins: {}'.format( len(conda_packages), ' '.join(conda_packages) @@ -338,7 +345,7 @@ def run_plugin_actions(plugins): conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) # Install pip packages - pip_packages = list(set(itertools.chain(*pm.hook.tljh_extra_user_pip_packages()))) + pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) if pip_packages: logger.info('Installing {} pip packages collected from plugins: {}'.format( len(pip_packages), ' '.join(pip_packages) @@ -346,6 +353,23 @@ def run_plugin_actions(plugins): conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages) +def ensure_config_yaml(plugin_manager): + """ + Ensure we have a config.yaml present + """ + if os.path.exists(CONFIG_FILE): + with open(CONFIG_FILE, 'r') as f: + config = rt_yaml.load(f) + else: + config = {} + + hook = plugin_manager.hook + hook.tljh_config_post_install(config=config) + + with open(CONFIG_FILE, 'w+') as f: + rt_yaml.dump(config, f) + + def main(): argparser = argparse.ArgumentParser() argparser.add_argument( @@ -365,6 +389,8 @@ def main(): args = argparser.parse_args() + pm = setup_plugins(args.plugin) + ensure_admins(args.admin) ensure_usergroups() @@ -374,12 +400,13 @@ def main(): ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) ensure_chp_package(HUB_ENV_PREFIX) + ensure_config_yaml(pm) ensure_jupyterhub_service(HUB_ENV_PREFIX) ensure_jupyterhub_running() ensure_symlinks(HUB_ENV_PREFIX) # Run installer plugins last - run_plugin_actions(args.plugin) + run_plugin_actions(pm, args.plugin) logger.info("Done!") From 8b2980a20a1c37fa8581e1ee150688e9d1b4b18e Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 12 Aug 2018 09:47:32 -0700 Subject: [PATCH 3/6] Add tests for the plugin mechanism --- .circleci/config.yml | 8 +++- integration-tests/plugins/simplest/setup.py | 9 ++++ .../plugins/simplest/tljh_simplest.py | 33 +++++++++++++ integration-tests/test_simplest_plugin.py | 47 +++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 integration-tests/plugins/simplest/setup.py create mode 100644 integration-tests/plugins/simplest/tljh_simplest.py create mode 100644 integration-tests/test_simplest_plugin.py diff --git a/.circleci/config.yml b/.circleci/config.yml index fe058b2..ed9fac9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,10 +68,16 @@ jobs: .circleci/integration-test.py build-image - run: - name: Run basic tests tests + name: Run basic tests command: | .circleci/integration-test.py run-test basic-tests test_hub.py test_install.py test_extensions.py + - run: + name: Run plugin tests + command: | + .circleci/integration-test.py run-test \ + --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ + plugins test_simplest_plugin.py documentation: diff --git a/integration-tests/plugins/simplest/setup.py b/integration-tests/plugins/simplest/setup.py new file mode 100644 index 0000000..9a34d26 --- /dev/null +++ b/integration-tests/plugins/simplest/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup( + name="tljh-simplest", + entry_points={"tljh": ["simplest = tljh_simplest"]}, + py_modules=["tljh_simplest"], +) + + diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py new file mode 100644 index 0000000..3d548e7 --- /dev/null +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -0,0 +1,33 @@ +""" +Simplest plugin that excercises all the hooks +""" +from tljh.hooks import hookimpl + + +@hookimpl +def tljh_extra_user_conda_packages(): + return [ + 'hypothesis', + ] + + +@hookimpl +def tljh_extra_user_pip_packages(): + return [ + 'django', + ] + + +@hookimpl +def tljh_extra_apt_packages(): + return [ + 'sl', + ] + + +@hookimpl +def tljh_config_post_install(config): + # Put an arbitrary marker we can test for + config['simplest_plugin'] = { + 'present': True + } \ No newline at end of file diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py new file mode 100644 index 0000000..0064616 --- /dev/null +++ b/integration-tests/test_simplest_plugin.py @@ -0,0 +1,47 @@ +""" +Test simplest plugin +""" +from ruamel.yaml import YAML +import os +import subprocess + +yaml = YAML(typ='rt') + + +def test_apt_packages(): + """ + Test extra apt packages are installed + """ + assert os.path.exists('/usr/bin/sl') + + +def test_pip_packages(): + """ + Test extra user pip packages are installed + """ + subprocess.check_call([ + '/opt/tljh/user/bin/python3', + '-c', + 'import django' + ]) + + +def test_conda_packages(): + """ + Test extra user conda packages are installed + """ + subprocess.check_call([ + '/opt/tljh/user/bin/python3', + '-c', + 'import hypothesis' + ]) + + +def test_config_hook(): + """ + Check config changes are present + """ + with open('/opt/tljh/config.yaml') as f: + data = yaml.load(f) + + assert data['simplest_plugin']['present'] \ No newline at end of file From 5a2ec9e517a77ad00cb1ad146d0a6fc700e24a8f Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 12 Aug 2018 09:53:49 -0700 Subject: [PATCH 4/6] Don't fail silently when integration-test commands fail This will lead to tests reported as passing even when they are actually failing. --- .circleci/integration-test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/integration-test.py b/.circleci/integration-test.py index fbd1272..93d0e2e 100755 --- a/.circleci/integration-test.py +++ b/.circleci/integration-test.py @@ -57,12 +57,7 @@ def run_container_command(container_name, cmd): 'docker', 'exec', '-it', container_name, '/bin/bash', '-c', cmd - ]) - - if proc.returncode != 0: - # Don't throw if command fails. This lets us continue next parts - # of tests. Not entirely sure this is the right thing to do though! - print(f'command {cmd} exited with return code {proc.returncode}') + ], check=True) def copy_to_container(container_name, src_path, dest_path): From c2a9bda74a139d1c87bfc8562e76e4b1a183c4fa Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 12 Aug 2018 09:55:20 -0700 Subject: [PATCH 5/6] Fix path to where sl is installed in plugin test I guess sl is *technically* a game. I'm sure there was a long winded argument about where it should be in a mailing list somewhere. --- integration-tests/test_simplest_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 0064616..c35de5d 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -12,7 +12,7 @@ def test_apt_packages(): """ Test extra apt packages are installed """ - assert os.path.exists('/usr/bin/sl') + assert os.path.exists('/usr/games/sl') def test_pip_packages(): From 3dd51fafae18f0002eec8eb051aa354adaf6bb83 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 12 Aug 2018 17:02:24 -0700 Subject: [PATCH 6/6] Add docs on writing & using plugins --- docs/contributing/plugins.rst | 119 +++++++++++++++++++++++++++ docs/index.rst | 1 + docs/topic/customizing-installer.rst | 29 +++++++ 3 files changed, 149 insertions(+) create mode 100644 docs/contributing/plugins.rst diff --git a/docs/contributing/plugins.rst b/docs/contributing/plugins.rst new file mode 100644 index 0000000..97e4eaa --- /dev/null +++ b/docs/contributing/plugins.rst @@ -0,0 +1,119 @@ +.. _contributing/plugins: + +============ +TLJH Plugins +============ + +TLJH plugins are the official way to make customized 'spins' or 'stacks' +with TLJH as the base. For example, the earth sciences community can make +a plugin that installs commonly used packages, set up authentication +and pre-download useful datasets. The mybinder.org community can +make a plugin that gives you a single-node, single-repository mybinder.org. +Plugins are very powerful, so the possibilities are endless. + +Design +====== + +`pluggy `_ is used to implement +plugin functionality. TLJH exposes specific **hooks** that your plugin +can provide implementations for. This allows us to have specific hook +points in the application that can be explicitly extended by plugins, +balancing the need to change TLJH internals in the future with the +stability required for a good plugin ecosystem. + +Writing a simple plugins +======================== + +We shall try to write a simple plugin that installs a few libraries, +and use it to explain how the plugin mechanism works. We shall call +this plugin ``tljh-simple``. + +Plugin directory layout +----------------------- + +We recommend creating a new git repo for your plugin. Plugins are +normal python packages - however, since they are usually simpler, +we recommend they live in one file. + +For ``tljh-simple``, the repository's structure should look like: + +.. code-block:: none + + tljh_simple: + - tljh_simple.py + - setup.py + - README.md + - LICENSE + +The ``README.md`` (or ``README.rst`` file) contains human readable +information about what your plugin does for your users. ``LICENSE`` +specifies the license used by your plugin - we recommend the +3-Clause BSD License, since that is what is used by TLJH itself. + +``setup.py`` - metadata & registration +-------------------------------------- + +``setup.py`` marks this as a python package, and contains metadata +about the package itself. It should look something like: + +.. code-block:: python + + from setuptools import setup + + setup( + name="tljh-simple", + author="YuviPanda", + version="0.1", + license="3-clause BSD", + url='https://github.com/yuvipanda/tljh-simple', + entry_points={"tljh": ["simple = tljh_simple"]}, + py_modules=["tljh_simple"], + ) + + +This is a mostly standard ``setup.py`` file. ``entry_points={"tljh": ["simple = tljh_simple]}`` +'registers' the module ``tljh_simple`` (in file ``tljh_simple.py``) with TLJH as a plugin. + +``tljh_simple.py`` - implementation +----------------------------------- + +In ``tljh_simple.py``, you provide implementations for whichever hooks +you want to extend. + +A hook implementation is a function that has the following characteristics: + +#. Has same name as the hook +#. Accepts some or all of the parameters defined for the hook +#. Is decorated with the ``hookimpl`` decorator function, imported from + ``tljh.hooks``. + +The current list of available hooks and when they are called can be +seen in ```tljh/hooks.py`` `_ +in the source repository. + + +This example provides an implementation for the ``tljh_extra_user_conda_packages`` +hook, which can return a list of conda packages that'll be installed in users' +environment from conda-forge. + +.. code-block:: python + + from tljh.hooks import hookimpl + + @hookimpl + def tljh_extra_user_conda_packages(): + return [ + 'xarray', + 'iris', + 'dask', + ] + + +Publishing plugins +================== + +Plugins are python packages and should be published on PyPI. Users +can also install them directly from GitHub - although this is +not good long term practice. + +The python package should be named ``tljh-``. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index a03589e..b84366b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -127,3 +127,4 @@ to people contributing in various ways. contributing/code-review contributing/dev-setup contributing/tests + contributing/plugins \ No newline at end of file diff --git a/docs/topic/customizing-installer.rst b/docs/topic/customizing-installer.rst index 30f1f99..359baea 100644 --- a/docs/topic/customizing-installer.rst +++ b/docs/topic/customizing-installer.rst @@ -56,3 +56,32 @@ will fail. When pointing to a file on GitHub, make sure to use the 'Raw' version. It should point to ``raw.githubusercontent.com``, not ``github.com``. + +Installing TLJH plugins +======================= + +The Littlest JupyterHub can install additional *plugins* that provide additional +features. They are most commonly used to install a particular *stack* - such as +the `PANGEO Stack `_ for earth sciences +research, a stack for a praticular class, etc. + +``--plugin `` installs and activates a plugin. You can pass it +however many times you want. Since plugins are distributed as python packages, +```` can be anything that can be passed to ``pip install`` - +``plugin-name-on-pypy==`` and ``git+https://github.com/user/repo@tag`` +are the most popular ones. Specifying a version or tag is highly recommended. + +For example, to install the PANGEO Plugin version 0.1 in your new TLJH install, +you would use: + +.. code-block:: bash + + curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \ + | sudo python3 - \ + --plugin git+https://github.com/yuvipanda/tljh-pangeo@v0.1 + + +.. note:: + + Plugins are extremely powerful and can do a large number of arbitrary things. + Only install plugins you trust. \ No newline at end of file