Merge pull request #127 from yuvipanda/pluggy

Add plugin support to the installer

.circleci/config.yml

@@ -68,10 +68,16 @@ jobs:
                     .circleci/integration-test.py build-image
 
             - run:
-                name: Run basic tests tests
+                name: Run basic tests
                 command: |
                     .circleci/integration-test.py run-test basic-tests test_hub.py test_install.py test_extensions.py
 
+            - run:
+                name: Run plugin tests
+                command: |
+                    .circleci/integration-test.py run-test \
+                        --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \
+                        plugins test_simplest_plugin.py
 
 
     documentation:

.circleci/integration-test.py

@@ -57,12 +57,7 @@ def run_container_command(container_name, cmd):
         'docker', 'exec',
         '-it', container_name,
         '/bin/bash', '-c', cmd
-    ])
-
-    if proc.returncode != 0:
-        # Don't throw if command fails. This lets us continue next parts
-        # of tests. Not entirely sure this is the right thing to do though!
-        print(f'command {cmd} exited with return code {proc.returncode}')
+    ], check=True)
 
 
 def copy_to_container(container_name, src_path, dest_path):

docs/contributing/plugins.rst

@@ -0,0 +1,119 @@
+.. _contributing/plugins:
+
+============
+TLJH Plugins
+============
+
+TLJH plugins are the official way to make customized 'spins' or 'stacks' 
+with TLJH as the base. For example, the earth sciences community can make
+a plugin that installs commonly used packages, set up authentication
+and pre-download useful datasets. The mybinder.org community can
+make a plugin that gives you a single-node, single-repository mybinder.org.
+Plugins are very powerful, so the possibilities are endless.
+
+Design
+======
+
+`pluggy <https://github.com/pytest-dev/pluggy>`_ is used to implement
+plugin functionality. TLJH exposes specific **hooks** that your plugin
+can provide implementations for. This allows us to have specific hook
+points in the application that can be explicitly extended by plugins,
+balancing the need to change TLJH internals in the future with the
+stability required for a good plugin ecosystem. 
+
+Writing a simple plugins
+========================
+
+We shall try to write a simple plugin that installs a few libraries,
+and use it to explain how the plugin mechanism works. We shall call
+this plugin ``tljh-simple``.
+
+Plugin directory layout
+-----------------------
+
+We recommend creating a new git repo for your plugin. Plugins are
+normal python packages - however, since they are usually simpler,
+we recommend they live in one file.
+
+For ``tljh-simple``, the repository's structure should look like:
+
+.. code-block:: none
+   
+   tljh_simple:
+    - tljh_simple.py
+    - setup.py
+    - README.md
+    - LICENSE
+
+The ``README.md`` (or ``README.rst`` file) contains human readable
+information about what your plugin does for your users. ``LICENSE``
+specifies the license used by your plugin - we recommend the 
+3-Clause BSD License, since that is what is used by TLJH itself.
+
+``setup.py`` - metadata & registration
+--------------------------------------
+
+``setup.py`` marks this as a python package, and contains metadata
+about the package itself. It should look something like:
+
+.. code-block:: python
+
+    from setuptools import setup
+
+    setup(
+        name="tljh-simple",
+        author="YuviPanda",
+        version="0.1",
+        license="3-clause BSD",
+        url='https://github.com/yuvipanda/tljh-simple',
+        entry_points={"tljh": ["simple = tljh_simple"]},
+        py_modules=["tljh_simple"],
+    )
+
+
+This is a mostly standard ``setup.py`` file. ``entry_points={"tljh": ["simple = tljh_simple]}``
+'registers' the module ``tljh_simple`` (in file ``tljh_simple.py``) with TLJH as a plugin.
+
+``tljh_simple.py`` - implementation
+-----------------------------------
+
+In ``tljh_simple.py``, you provide implementations for whichever hooks
+you want to extend. 
+
+A hook implementation is a function that has the following characteristics:
+
+#. Has same name as the hook
+#. Accepts some or all of the parameters defined for the hook
+#. Is decorated with the ``hookimpl`` decorator function, imported from
+   ``tljh.hooks``.
+
+The current list of available hooks and when they are called can be
+seen in ```tljh/hooks.py`` <https://github.com/jupyterhub/the-littlest-jupyterhub/blob/master/tljh/hooks.py>`_
+in the source repository.
+
+
+This example provides an implementation for the ``tljh_extra_user_conda_packages``
+hook, which can return a list of conda packages that'll be installed in users' 
+environment from conda-forge.
+
+.. code-block:: python
+
+    from tljh.hooks import hookimpl
+
+    @hookimpl
+    def tljh_extra_user_conda_packages():
+        return [
+            'xarray',
+            'iris',
+            'dask',
+        ]
+
+
+Publishing plugins
+==================
+
+Plugins are python packages and should be published on PyPI. Users
+can also install them directly from GitHub - although this is
+not good long term practice. 
+
+The python package should be named ``tljh-<pluginname>``.

docs/index.rst

@@ -127,3 +127,4 @@ to people contributing in various ways.
    contributing/code-review
    contributing/dev-setup
    contributing/tests
+   contributing/plugins

docs/topic/customizing-installer.rst

@@ -56,3 +56,32 @@ will fail.
 
    When pointing to a file on GitHub, make sure to use the 'Raw' version. It should point to
    ``raw.githubusercontent.com``, not ``github.com``.
+
+Installing TLJH plugins
+=======================
+
+The Littlest JupyterHub can install additional *plugins* that provide additional
+features. They are most commonly used to install a particular *stack* - such as
+the `PANGEO Stack <https://github.com/yuvipanda/tljh-pangeo>`_ for earth sciences
+research, a stack for a praticular class, etc.
+
+``--plugin <plugin-to-install>`` installs and activates a plugin. You can pass it
+however many times you want. Since plugins are distributed as python packages,
+``<plugin-to-install>`` can be anything that can be passed to ``pip install`` -
+``plugin-name-on-pypy==<version>`` and ``git+https://github.com/user/repo@tag``
+are the most popular ones. Specifying a version or tag is highly recommended.
+
+For example, to install the PANGEO Plugin version 0.1 in your new TLJH install,
+you would use:
+
+.. code-block:: bash
+
+   curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
+    | sudo python3 - \
+      --plugin git+https://github.com/yuvipanda/tljh-pangeo@v0.1
+
+
+.. note::
+
+   Plugins are extremely powerful and can do a large number of arbitrary things.
+   Only install plugins you trust.

integration-tests/plugins/simplest/setup.py

@@ -0,0 +1,9 @@
+from setuptools import setup
+
+setup(
+    name="tljh-simplest",
+    entry_points={"tljh": ["simplest = tljh_simplest"]},
+    py_modules=["tljh_simplest"],
+)
+
+

integration-tests/plugins/simplest/tljh_simplest.py

@@ -0,0 +1,33 @@
+"""
+Simplest plugin that excercises all the hooks
+"""
+from tljh.hooks import hookimpl
+
+
+@hookimpl
+def tljh_extra_user_conda_packages():
+    return [
+        'hypothesis',
+    ]
+
+
+@hookimpl
+def tljh_extra_user_pip_packages():
+    return [
+        'django',
+    ]
+
+
+@hookimpl
+def tljh_extra_apt_packages():
+    return [
+        'sl',
+    ]
+
+
+@hookimpl
+def tljh_config_post_install(config):
+    # Put an arbitrary marker we can test for
+    config['simplest_plugin'] = {
+        'present': True
+    }

integration-tests/test_simplest_plugin.py

@@ -0,0 +1,47 @@
+"""
+Test simplest plugin
+"""
+from ruamel.yaml import YAML
+import os
+import subprocess
+
+yaml = YAML(typ='rt')
+
+
+def test_apt_packages():
+    """
+    Test extra apt packages are installed
+    """
+    assert os.path.exists('/usr/games/sl')
+
+
+def test_pip_packages():
+    """
+    Test extra user pip packages are installed
+    """
+    subprocess.check_call([
+        '/opt/tljh/user/bin/python3',
+        '-c',
+        'import django'
+    ])
+
+
+def test_conda_packages():
+    """
+    Test extra user conda packages are installed
+    """
+    subprocess.check_call([
+        '/opt/tljh/user/bin/python3',
+        '-c',
+        'import hypothesis'
+    ])
+
+
+def test_config_hook():
+    """
+    Check config changes are present
+    """
+    with open('/opt/tljh/config.yaml') as f:
+        data = yaml.load(f)
+
+    assert data['simplest_plugin']['present']

setup.py

@@ -14,6 +14,7 @@ setup(
         'pyyaml==3.*',
         'ruamel.yaml==0.15.*',
         'jinja2',
+        'pluggy>0.7<1.0'
     ],
     entry_points={
         'console_scripts': [

tljh/hooks.py

@@ -0,0 +1,46 @@
+"""
+Hook specifications that pluggy plugins can override
+"""
+import pluggy
+
+hookspec = pluggy.HookspecMarker('tljh')
+hookimpl = pluggy.HookimplMarker('tljh')
+
+
+@hookspec
+def tljh_extra_user_conda_packages():
+    """
+    Return list of extra conda packages to install in user environment.
+    """
+    pass
+
+
+@hookspec
+def tljh_extra_user_pip_packages():
+    """
+    Return list of extra pip packages to install in user environment.
+    """
+    pass
+
+
+@hookspec
+def tljh_extra_apt_packages():
+    """
+    Return list of extra apt packages to install in the user environment.
+
+    These will be installed before additional pip or conda packages.
+    """
+    pass
+
+
+@hookspec
+def tljh_config_post_install(config):
+    """
+    Modify on-disk tljh-config after installation.
+
+    config is a dict-like object that should be modified
+    in-place. The contents of the on-disk config.yaml will
+    be the serialized contents of config, so try to not
+    overwrite anything the user might have explicitly set.
+    """
+    pass

tljh/installer.py

@@ -2,16 +2,18 @@ import argparse
 import os
 import secrets
 import subprocess
+import itertools
 import sys
 import time
 import logging
 from urllib.error import HTTPError
 from urllib.request import urlopen, URLError
+import pluggy
 
 from ruamel.yaml import YAML
 
-from tljh import conda, systemd, traefik, user, apt
-from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR
+from tljh import conda, systemd, traefik, user, apt, hooks
+from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR, CONFIG_FILE
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
@@ -306,6 +308,68 @@ def ensure_symlinks(prefix):
     os.symlink(tljh_config_src, tljh_config_dest)
 
 
+def setup_plugins(plugins):
+    """
+    Install plugins & setup a pluginmanager
+    """
+    # Install plugins
+    if plugins:
+        conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins)
+
+    # Set up plugin infrastructure
+    pm = pluggy.PluginManager('tljh')
+    pm.add_hookspecs(hooks)
+    pm.load_setuptools_entrypoints('tljh')
+
+    return pm
+
+def run_plugin_actions(plugin_manager, plugins):
+    """
+    Run installer hooks defined in plugins
+    """
+    hook = plugin_manager.hook
+    # Install apt packages
+    apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages())))
+    if apt_packages:
+        logger.info('Installing {} apt packages collected from plugins: {}'.format(
+            len(apt_packages), ' '.join(apt_packages)
+        ))
+        apt.install_packages(apt_packages)
+
+    # Install conda packages
+    conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages())))
+    if conda_packages:
+        logger.info('Installing {} conda packages collected from plugins: {}'.format(
+            len(conda_packages), ' '.join(conda_packages)
+        ))
+        conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages)
+
+    # Install pip packages
+    pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages())))
+    if pip_packages:
+        logger.info('Installing {} pip packages collected from plugins: {}'.format(
+            len(pip_packages), ' '.join(pip_packages)
+        ))
+        conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages)
+
+
+def ensure_config_yaml(plugin_manager):
+    """
+    Ensure we have a config.yaml present
+    """
+    if os.path.exists(CONFIG_FILE):
+        with open(CONFIG_FILE, 'r') as f:
+            config = rt_yaml.load(f)
+    else:
+        config = {}
+
+    hook = plugin_manager.hook
+    hook.tljh_config_post_install(config=config)
+
+    with open(CONFIG_FILE, 'w+') as f:
+        rt_yaml.dump(config, f)
+
+
 def main():
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
@@ -317,10 +381,15 @@ def main():
         '--user-requirements-txt-url',
         help='URL to a requirements.txt file that should be installed in the user enviornment'
     )
+    argparser.add_argument(
+        '--plugin',
+        nargs='*',
+        help='Plugin pip-specs to install'
+    )
 
     args = argparser.parse_args()
 
-
+    pm = setup_plugins(args.plugin)
 
     ensure_admins(args.admin)
 
@@ -331,10 +400,14 @@ def main():
     ensure_node()
     ensure_jupyterhub_package(HUB_ENV_PREFIX)
     ensure_chp_package(HUB_ENV_PREFIX)
+    ensure_config_yaml(pm)
     ensure_jupyterhub_service(HUB_ENV_PREFIX)
     ensure_jupyterhub_running()
     ensure_symlinks(HUB_ENV_PREFIX)
 
+    # Run installer plugins last
+    run_plugin_actions(pm, args.plugin)
+
     logger.info("Done!")