diff --git a/.circleci/config.yml b/.circleci/config.yml index fe058b2..ed9fac9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,10 +68,16 @@ jobs: .circleci/integration-test.py build-image - run: - name: Run basic tests tests + name: Run basic tests command: | .circleci/integration-test.py run-test basic-tests test_hub.py test_install.py test_extensions.py + - run: + name: Run plugin tests + command: | + .circleci/integration-test.py run-test \ + --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ + plugins test_simplest_plugin.py documentation: diff --git a/.circleci/integration-test.py b/.circleci/integration-test.py index fbd1272..93d0e2e 100755 --- a/.circleci/integration-test.py +++ b/.circleci/integration-test.py @@ -57,12 +57,7 @@ def run_container_command(container_name, cmd): 'docker', 'exec', '-it', container_name, '/bin/bash', '-c', cmd - ]) - - if proc.returncode != 0: - # Don't throw if command fails. This lets us continue next parts - # of tests. Not entirely sure this is the right thing to do though! - print(f'command {cmd} exited with return code {proc.returncode}') + ], check=True) def copy_to_container(container_name, src_path, dest_path): diff --git a/docs/contributing/plugins.rst b/docs/contributing/plugins.rst new file mode 100644 index 0000000..97e4eaa --- /dev/null +++ b/docs/contributing/plugins.rst @@ -0,0 +1,119 @@ +.. _contributing/plugins: + +============ +TLJH Plugins +============ + +TLJH plugins are the official way to make customized 'spins' or 'stacks' +with TLJH as the base. For example, the earth sciences community can make +a plugin that installs commonly used packages, set up authentication +and pre-download useful datasets. The mybinder.org community can +make a plugin that gives you a single-node, single-repository mybinder.org. +Plugins are very powerful, so the possibilities are endless. + +Design +====== + +`pluggy `_ is used to implement +plugin functionality. TLJH exposes specific **hooks** that your plugin +can provide implementations for. This allows us to have specific hook +points in the application that can be explicitly extended by plugins, +balancing the need to change TLJH internals in the future with the +stability required for a good plugin ecosystem. + +Writing a simple plugins +======================== + +We shall try to write a simple plugin that installs a few libraries, +and use it to explain how the plugin mechanism works. We shall call +this plugin ``tljh-simple``. + +Plugin directory layout +----------------------- + +We recommend creating a new git repo for your plugin. Plugins are +normal python packages - however, since they are usually simpler, +we recommend they live in one file. + +For ``tljh-simple``, the repository's structure should look like: + +.. code-block:: none + + tljh_simple: + - tljh_simple.py + - setup.py + - README.md + - LICENSE + +The ``README.md`` (or ``README.rst`` file) contains human readable +information about what your plugin does for your users. ``LICENSE`` +specifies the license used by your plugin - we recommend the +3-Clause BSD License, since that is what is used by TLJH itself. + +``setup.py`` - metadata & registration +-------------------------------------- + +``setup.py`` marks this as a python package, and contains metadata +about the package itself. It should look something like: + +.. code-block:: python + + from setuptools import setup + + setup( + name="tljh-simple", + author="YuviPanda", + version="0.1", + license="3-clause BSD", + url='https://github.com/yuvipanda/tljh-simple', + entry_points={"tljh": ["simple = tljh_simple"]}, + py_modules=["tljh_simple"], + ) + + +This is a mostly standard ``setup.py`` file. ``entry_points={"tljh": ["simple = tljh_simple]}`` +'registers' the module ``tljh_simple`` (in file ``tljh_simple.py``) with TLJH as a plugin. + +``tljh_simple.py`` - implementation +----------------------------------- + +In ``tljh_simple.py``, you provide implementations for whichever hooks +you want to extend. + +A hook implementation is a function that has the following characteristics: + +#. Has same name as the hook +#. Accepts some or all of the parameters defined for the hook +#. Is decorated with the ``hookimpl`` decorator function, imported from + ``tljh.hooks``. + +The current list of available hooks and when they are called can be +seen in ```tljh/hooks.py`` `_ +in the source repository. + + +This example provides an implementation for the ``tljh_extra_user_conda_packages`` +hook, which can return a list of conda packages that'll be installed in users' +environment from conda-forge. + +.. code-block:: python + + from tljh.hooks import hookimpl + + @hookimpl + def tljh_extra_user_conda_packages(): + return [ + 'xarray', + 'iris', + 'dask', + ] + + +Publishing plugins +================== + +Plugins are python packages and should be published on PyPI. Users +can also install them directly from GitHub - although this is +not good long term practice. + +The python package should be named ``tljh-``. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index a03589e..b84366b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -127,3 +127,4 @@ to people contributing in various ways. contributing/code-review contributing/dev-setup contributing/tests + contributing/plugins \ No newline at end of file diff --git a/docs/topic/customizing-installer.rst b/docs/topic/customizing-installer.rst index 30f1f99..359baea 100644 --- a/docs/topic/customizing-installer.rst +++ b/docs/topic/customizing-installer.rst @@ -56,3 +56,32 @@ will fail. When pointing to a file on GitHub, make sure to use the 'Raw' version. It should point to ``raw.githubusercontent.com``, not ``github.com``. + +Installing TLJH plugins +======================= + +The Littlest JupyterHub can install additional *plugins* that provide additional +features. They are most commonly used to install a particular *stack* - such as +the `PANGEO Stack `_ for earth sciences +research, a stack for a praticular class, etc. + +``--plugin `` installs and activates a plugin. You can pass it +however many times you want. Since plugins are distributed as python packages, +```` can be anything that can be passed to ``pip install`` - +``plugin-name-on-pypy==`` and ``git+https://github.com/user/repo@tag`` +are the most popular ones. Specifying a version or tag is highly recommended. + +For example, to install the PANGEO Plugin version 0.1 in your new TLJH install, +you would use: + +.. code-block:: bash + + curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \ + | sudo python3 - \ + --plugin git+https://github.com/yuvipanda/tljh-pangeo@v0.1 + + +.. note:: + + Plugins are extremely powerful and can do a large number of arbitrary things. + Only install plugins you trust. \ No newline at end of file diff --git a/integration-tests/plugins/simplest/setup.py b/integration-tests/plugins/simplest/setup.py new file mode 100644 index 0000000..9a34d26 --- /dev/null +++ b/integration-tests/plugins/simplest/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup( + name="tljh-simplest", + entry_points={"tljh": ["simplest = tljh_simplest"]}, + py_modules=["tljh_simplest"], +) + + diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py new file mode 100644 index 0000000..3d548e7 --- /dev/null +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -0,0 +1,33 @@ +""" +Simplest plugin that excercises all the hooks +""" +from tljh.hooks import hookimpl + + +@hookimpl +def tljh_extra_user_conda_packages(): + return [ + 'hypothesis', + ] + + +@hookimpl +def tljh_extra_user_pip_packages(): + return [ + 'django', + ] + + +@hookimpl +def tljh_extra_apt_packages(): + return [ + 'sl', + ] + + +@hookimpl +def tljh_config_post_install(config): + # Put an arbitrary marker we can test for + config['simplest_plugin'] = { + 'present': True + } \ No newline at end of file diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py new file mode 100644 index 0000000..c35de5d --- /dev/null +++ b/integration-tests/test_simplest_plugin.py @@ -0,0 +1,47 @@ +""" +Test simplest plugin +""" +from ruamel.yaml import YAML +import os +import subprocess + +yaml = YAML(typ='rt') + + +def test_apt_packages(): + """ + Test extra apt packages are installed + """ + assert os.path.exists('/usr/games/sl') + + +def test_pip_packages(): + """ + Test extra user pip packages are installed + """ + subprocess.check_call([ + '/opt/tljh/user/bin/python3', + '-c', + 'import django' + ]) + + +def test_conda_packages(): + """ + Test extra user conda packages are installed + """ + subprocess.check_call([ + '/opt/tljh/user/bin/python3', + '-c', + 'import hypothesis' + ]) + + +def test_config_hook(): + """ + Check config changes are present + """ + with open('/opt/tljh/config.yaml') as f: + data = yaml.load(f) + + assert data['simplest_plugin']['present'] \ No newline at end of file diff --git a/setup.py b/setup.py index f50745e..e01f4e1 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ setup( 'pyyaml==3.*', 'ruamel.yaml==0.15.*', 'jinja2', + 'pluggy>0.7<1.0' ], entry_points={ 'console_scripts': [ diff --git a/tljh/hooks.py b/tljh/hooks.py new file mode 100644 index 0000000..8cd6d56 --- /dev/null +++ b/tljh/hooks.py @@ -0,0 +1,46 @@ +""" +Hook specifications that pluggy plugins can override +""" +import pluggy + +hookspec = pluggy.HookspecMarker('tljh') +hookimpl = pluggy.HookimplMarker('tljh') + + +@hookspec +def tljh_extra_user_conda_packages(): + """ + Return list of extra conda packages to install in user environment. + """ + pass + + +@hookspec +def tljh_extra_user_pip_packages(): + """ + Return list of extra pip packages to install in user environment. + """ + pass + + +@hookspec +def tljh_extra_apt_packages(): + """ + Return list of extra apt packages to install in the user environment. + + These will be installed before additional pip or conda packages. + """ + pass + + +@hookspec +def tljh_config_post_install(config): + """ + Modify on-disk tljh-config after installation. + + config is a dict-like object that should be modified + in-place. The contents of the on-disk config.yaml will + be the serialized contents of config, so try to not + overwrite anything the user might have explicitly set. + """ + pass \ No newline at end of file diff --git a/tljh/installer.py b/tljh/installer.py index fa2b838..301e83c 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -2,16 +2,18 @@ import argparse import os import secrets import subprocess +import itertools import sys import time import logging from urllib.error import HTTPError from urllib.request import urlopen, URLError +import pluggy from ruamel.yaml import YAML -from tljh import conda, systemd, traefik, user, apt -from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR +from tljh import conda, systemd, traefik, user, apt, hooks +from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR, CONFIG_FILE HERE = os.path.abspath(os.path.dirname(__file__)) @@ -306,6 +308,68 @@ def ensure_symlinks(prefix): os.symlink(tljh_config_src, tljh_config_dest) +def setup_plugins(plugins): + """ + Install plugins & setup a pluginmanager + """ + # Install plugins + if plugins: + conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) + + # Set up plugin infrastructure + pm = pluggy.PluginManager('tljh') + pm.add_hookspecs(hooks) + pm.load_setuptools_entrypoints('tljh') + + return pm + +def run_plugin_actions(plugin_manager, plugins): + """ + Run installer hooks defined in plugins + """ + hook = plugin_manager.hook + # Install apt packages + apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages()))) + if apt_packages: + logger.info('Installing {} apt packages collected from plugins: {}'.format( + len(apt_packages), ' '.join(apt_packages) + )) + apt.install_packages(apt_packages) + + # Install conda packages + conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) + if conda_packages: + logger.info('Installing {} conda packages collected from plugins: {}'.format( + len(conda_packages), ' '.join(conda_packages) + )) + conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) + + # Install pip packages + pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) + if pip_packages: + logger.info('Installing {} pip packages collected from plugins: {}'.format( + len(pip_packages), ' '.join(pip_packages) + )) + conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages) + + +def ensure_config_yaml(plugin_manager): + """ + Ensure we have a config.yaml present + """ + if os.path.exists(CONFIG_FILE): + with open(CONFIG_FILE, 'r') as f: + config = rt_yaml.load(f) + else: + config = {} + + hook = plugin_manager.hook + hook.tljh_config_post_install(config=config) + + with open(CONFIG_FILE, 'w+') as f: + rt_yaml.dump(config, f) + + def main(): argparser = argparse.ArgumentParser() argparser.add_argument( @@ -317,10 +381,15 @@ def main(): '--user-requirements-txt-url', help='URL to a requirements.txt file that should be installed in the user enviornment' ) + argparser.add_argument( + '--plugin', + nargs='*', + help='Plugin pip-specs to install' + ) args = argparser.parse_args() - + pm = setup_plugins(args.plugin) ensure_admins(args.admin) @@ -331,10 +400,14 @@ def main(): ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) ensure_chp_package(HUB_ENV_PREFIX) + ensure_config_yaml(pm) ensure_jupyterhub_service(HUB_ENV_PREFIX) ensure_jupyterhub_running() ensure_symlinks(HUB_ENV_PREFIX) + # Run installer plugins last + run_plugin_actions(pm, args.plugin) + logger.info("Done!")