Merge pull request #1 from jupyterhub/master

update my fork to latest , current version

.circleci/config.yml

@@ -57,7 +57,7 @@ jobs:
             - run:
                 name: setup python3
                 command: |
-                    apk add --no-cache python3
+                    apk add --no-cache python3 pytest
 
             - checkout
             - setup_remote_docker
@@ -79,6 +79,12 @@ jobs:
                         --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \
                         plugins test_simplest_plugin.py
 
+            - run:
+                name: Run bootstrap checks
+                command: |
+                    py.test integration-tests/test_bootstrap.py
+
+
 
     documentation:
         docker:
@@ -123,6 +129,10 @@ jobs:
                     cd docs
                     make html
 
+            - store_artifacts:
+                path: docs/_build/html/
+                destination: html
+
 workflows:
     version: 2
     all-tests:

.circleci/integration-test.py

@@ -29,6 +29,10 @@ def run_systemd_image(image_name, container_name):
         '--mount', 'type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup',
         '--detach',
         '--name', container_name,
+        # This is the minimum VM size we support. JupyterLab extensions seem
+        # to need at least this much RAM to build. Boo?
+        # If we change this, need to change all other references to this number.
+        '--memory', '768M',
         image_name
     ])
 
@@ -108,7 +112,7 @@ def show_logs(container_name):
     )
     run_container_command(
         container_name,
-        'systemctl --no-pager status jupyterhub configurable-http-proxy'
+        'systemctl --no-pager status jupyterhub traefik'
     )
 
 def main():

.gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# OS X stuff
+.DS_Store

README.rst

@@ -6,7 +6,7 @@ The Littlest JupyterHub
    :target: https://circleci.com/gh/jupyterhub/the-littlest-jupyterhub
 .. image:: https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/master/graph/badge.svg
   :target: https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub
-.. image:: https://media.readthedocs.org/static/projects/badges/passing-flat.svg
+.. image:: https://readthedocs.org/projects/the-littlest-jupyterhub/badge/?version=latest
    :target: https://the-littlest-jupyterhub.readthedocs.io
 .. image:: https://badges.gitter.im/jupyterhub/jupyterhub.svg
    :target: https://gitter.im/jupyterhub/jupyterhub
@@ -16,10 +16,8 @@ to 1-50 users on a single server.
 
 Administrators who do not consider themselves 'system administrators' but would
 like to provide hosted Jupyter Notebooks for their students / users are the
-primary audience. All users get the same environment, and administrators can
-install libraries into this environment without any specialized knowledge.
-It provides all users with the same environment, and administrators can install
-libraries into this environment easily without any specialized knowledge.
+primary audience. All users are provided with the same environment, and administrators
+can easily install libraries into this environment without any specialized knowledge.
 
 See `this blog post <http://words.yuvi.in/post/the-littlest-jupyterhub/>`_ for
 more information.
@@ -27,24 +25,27 @@ more information.
 Development Status
 ==================
 
-This project is currently in **alpha** state. Most things work, but we might
-still make breaking changes that have no clear upgrade pathway. We are targeting
-a v0.1 release sometime in mid-August 2018. Follow `this milestone <https://github.com/jupyterhub/the-littlest-jupyterhub/milestone/1>`_
-to see progress towards the release!
+This project is currently in **beta** state. Folks have been using installations
+of TLJH for more than a year now to great success. While we try hard not to, we
+might still make breaking changes that have no clear upgrade pathway.
 
 Installation
 ============
 
 The Littlest JupyterHub (TLJH) can run on any server that is running at least
-Ubuntu 18.04. We have a bunch of tutorials to get you started!
+**Ubuntu 18.04**. Earlier versions of Ubuntu are not supported.
+We have a bunch of tutorials to get you started.
 
 - Tutorials to create a new server from scratch on a cloud provider & run TLJH
   on it. These are **recommended** if you do not have much experience setting up
   servers.
 
   - `Digital Ocean <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/digitalocean.html>`_
+  - `OVH <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/ovh.html>`_
   - `Google Cloud <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/google.html>`_
   - `Jetstream <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/jetstream.html>`_
+  - `Amazon Web Services <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/amazon.html>`_
+  - `Microsoft Azure <https://the-littlest-jupyterhub.readthedocs.io/en/latest/install/azure.html>`_
   - ... your favorite provider here, if you can contribute!
 
 - `Tutorial to install TLJH on an already running server you have root access to

bootstrap/bootstrap.py

@@ -8,32 +8,109 @@ This script is run as:
     curl <script-url> | sudo python3 -
 
 Constraints:
-  - Be compatible with Python 3.4 (since we support Ubuntu 16.04)
+  - Entire script should be compatible with Python 3.6 (We run on Ubuntu 18.04+)
+  - Script should parse in Python 3.4 (since we exit with useful error message on Ubuntu 14.04+)
   - Use stdlib modules only
 """
 import os
 import subprocess
 import sys
 import logging
+import shutil
 
+logger = logging.getLogger(__name__)
 
+def get_os_release_variable(key):
+    """
+    Return value for key from /etc/os-release
+
+    /etc/os-release is a bash file, so should use bash to parse it.
+
+    Returns empty string if key is not found.
+    """
+    return subprocess.check_output([
+        '/bin/bash', '-c',
+        "source /etc/os-release && echo ${{{key}}}".format(key=key)
+    ]).decode().strip()
+
+# Copied into tljh/utils.py. Make sure the copies are exactly the same!
+def run_subprocess(cmd, *args, **kwargs):
+    """
+    Run given cmd with smart output behavior.
+
+    If command succeeds, print output to debug logging.
+    If it fails, print output to info logging.
+
+    In TLJH, this sends successful output to the installer log,
+    and failed output directly to the user's screen
+    """
+    logger = logging.getLogger('tljh')
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs)
+    printable_command = ' '.join(cmd)
+    if proc.returncode != 0:
+        # Our process failed! Show output to the user
+        logger.error('Ran {command} with exit code {code}'.format(
+            command=printable_command, code=proc.returncode
+        ))
+        logger.error(proc.stdout.decode())
+        raise subprocess.CalledProcessError(cmd=cmd, returncode=proc.returncode)
+    else:
+        # This goes into installer.log
+        logger.debug('Ran {command} with exit code {code}'.format(
+            command=printable_command, code=proc.returncode
+        ))
+        # This produces multi line log output, unfortunately. Not sure how to fix.
+        # For now, prioritizing human readability over machine readability.
+        logger.debug(proc.stdout.decode())
+
+def validate_host():
+    """
+    Make sure TLJH is installable in current host
+    """
+    # Support only Ubuntu 18.04+
+    distro = get_os_release_variable('ID')
+    version = float(get_os_release_variable('VERSION_ID'))
+    if distro != 'ubuntu':
+        print('The Littlest JupyterHub currently supports Ubuntu Linux only')
+        sys.exit(1)
+    elif float(version) < 18.04:
+        print('The Littlest JupyterHub requires Ubuntu 18.04 or higher')
+        sys.exit(1)
+
+    if sys.version_info < (3, 5):
+        print("bootstrap.py must be run with at least Python 3.5")
+        sys.exit(1)
+
+    if not (shutil.which('systemd') and shutil.which('systemctl')):
+        print("Systemd is required to run TLJH")
+        # Only fail running inside docker if systemd isn't present
+        if os.path.exists('/.dockerenv'):
+            print("Running inside a docker container without systemd isn't supported")
+            print("We recommend against running a production TLJH instance inside a docker container")
+            print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html")
+        sys.exit(1)
 
 def main():
+    validate_host()
     install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
     hub_prefix = os.path.join(install_prefix, 'hub')
 
     # Set up logging to print to a file and to stderr
-    logger = logging.getLogger(__name__)
-
     os.makedirs(install_prefix, exist_ok=True)
-    file_logger = logging.FileHandler(os.path.join(install_prefix, 'installer.log'))
+    file_logger_path = os.path.join(install_prefix, 'installer.log')
+    file_logger = logging.FileHandler(file_logger_path)
+    # installer.log should be readable only by root
+    os.chmod(file_logger_path, 0o500)
+
     file_logger.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
+    file_logger.setLevel(logging.DEBUG)
     logger.addHandler(file_logger)
 
     stderr_logger = logging.StreamHandler()
     stderr_logger.setFormatter(logging.Formatter('%(message)s'))
+    stderr_logger.setLevel(logging.INFO)
     logger.addHandler(stderr_logger)
-    logger.setLevel(logging.INFO)
+    logger.setLevel(logging.DEBUG)
 
     logger.info('Checking if TLJH is already installed...')
     if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')):
@@ -42,11 +119,25 @@ def main():
     else:
         logger.info('Setting up hub environment')
         initial_setup = True
-        subprocess.check_output(['apt-get', 'update', '--yes'], stderr=subprocess.STDOUT)
-        subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv'], stderr=subprocess.STDOUT)
+        # Install software-properties-common, so we can get add-apt-repository
+        # That helps us make sure the universe repository is enabled, since
+        # that's where the python3-pip package lives. In some very minimal base
+        # VM images, it looks like the universe repository is disabled by default,
+        # causing bootstrapping to fail.
+        run_subprocess(['apt-get', 'update', '--yes'])
+        run_subprocess(['apt-get', 'install', '--yes', 'software-properties-common'])
+        run_subprocess(['add-apt-repository', 'universe'])
+
+        run_subprocess(['apt-get', 'update', '--yes'])
+        run_subprocess(['apt-get', 'install', '--yes', 
+            'python3',
+            'python3-venv',
+            'python3-pip',
+            'git'
+        ])
         logger.info('Installed python & virtual environment')
         os.makedirs(hub_prefix, exist_ok=True)
-        subprocess.check_output(['python3', '-m', 'venv', hub_prefix], stderr=subprocess.STDOUT)
+        run_subprocess(['python3', '-m', 'venv', hub_prefix])
         logger.info('Set up hub virtual environment')
 
     if initial_setup:
@@ -62,10 +153,10 @@ def main():
         'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git'
     )
 
-    subprocess.check_output([
+    run_subprocess([
         os.path.join(hub_prefix, 'bin', 'pip'),
         'install'
-    ] + pip_flags + [tljh_repo_path], stderr=subprocess.STDOUT)
+    ] + pip_flags + [tljh_repo_path])
     logger.info('Setup tljh package')
 
     logger.info('Starting TLJH installer...')

dev-requirements.txt

@@ -1,4 +1,5 @@
 pytest
 pytest-cov
+pytest-mock
 codecov
-pytoml
+pytoml

docs/_static/custom.css

@@ -1,3 +1,3 @@
-div.sphinxsidebarwrapper h1.logo {
-  font-size: 2.3rem;
+div.toctree-wrapper p.caption {
+  display: none;
 }

docs/conf.py

@@ -34,7 +34,13 @@ intersphinx_cache_limit = 90 # days
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
-html_theme = 'alabaster'
+import alabaster_jupyterhub
+
+html_theme = 'alabaster_jupyterhub'
+html_theme_path = [alabaster_jupyterhub.get_html_theme_path()]
+
+html_logo = 'images/logo/logo.png'
+html_favicon = 'images/logo/favicon.ico'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -50,6 +56,7 @@ if os.path.exists(os.path.join(here, '_static')):
 html_sidebars = {
     '**': [
         'about.html',
+        'globaltoc.html',
         'relations.html',
         'searchbox.html',
         'donate.html',
@@ -65,5 +72,9 @@ html_theme_options = {
     'github_user': 'jupyterhub',
     'github_repo': 'the-littlest-jupyterhub',
     'github_button': True,
-    'github_banner': True,
+    'github_banner': False,
+    'github_type': 'star',
 }
+
+def setup(app):
+    app.add_stylesheet('custom.css')

docs/contributing/dev-setup.rst

@@ -1,4 +1,4 @@
-.. _contributing_dev_setup:
+.. _contributing/dev-setup:
 
 ==================================
 Setting up Development Environment
@@ -14,14 +14,14 @@ The easiest & safest way to develop & test TLJH is with `Docker <https://www.doc
 
    .. code-block:: bash
 
-      sudo docker build -t tljh-systemd .
+      docker build -t tljh-systemd . -f integration-tests/Dockerfile
 
 #. Run a docker container with the image in the background, while bind mounting
    your TLJH repository under ``/srv/src``.
 
    .. code-block:: bash
 
-      sudo docker run \
+      docker run \
         --privileged \
         --detach \
         --name=tljh-dev \
@@ -33,7 +33,7 @@ The easiest & safest way to develop & test TLJH is with `Docker <https://www.doc
 
    .. code-block:: bash
 
-      sudo docker exec -it tljh-dev /bin/bash
+      docker exec -it tljh-dev /bin/bash
 
 #. Run the bootstrapper from inside the container (see step above):
    The container image is already set up to default to a ``dev`` install, so
@@ -41,7 +41,7 @@ The easiest & safest way to develop & test TLJH is with `Docker <https://www.doc
 
    .. code-block:: console
 
-      python3 /srv/src/bootstrap/bootstrap.py
+      python3 /srv/src/bootstrap/bootstrap.py --admin admin
 
    The primary hub environment will also be in your PATH already for convenience.
 

docs/contributing/docs.rst

@@ -48,11 +48,12 @@ documentation is transformed into HTML, PDF, and any other output format.
 __ http://sphinx-doc.org/
 __ http://docutils.sourceforge.net/
 
-To build the documentation locally, install Sphinx:
+To build the documentation locally, install the Sphinx dependencies:
 
 .. code-block:: console
 
-     $ pip install Sphinx
+     $ cd docs/
+     $ pip install -r requirements.txt
 
 Then from the ``docs`` directory, build the HTML:
 
@@ -60,6 +61,12 @@ Then from the ``docs`` directory, build the HTML:
 
      $ make html
 
+If you encounter this error, it's likely that you are running inside a virtual environment.
+
+.. code-block:: console
+
+   Error in "currentmodule" directive:
+
 To get started contributing, you'll want to read the :ref:`reStructuredText
 reference <sphinx:rst-index>`
 
@@ -134,6 +141,7 @@ The documentation is organized into several categories:
 Writing style
 =============
 
+Typically, documentation is written in second person, referring to the reader as “you”. 
 When using pronouns in reference to a hypothetical person, such as "a user with
 a running notebook", gender neutral pronouns (they/their/them) should be used.
 Instead of:
@@ -198,11 +206,18 @@ Our policy for new features is:
     All new features must have appropriate documentation before they
     can be merged.
 
-Minimizing images
-=================
+Choosing image size
+===================
 
-Optimize image compression where possible. For PNG files, use OptiPNG and
-AdvanceCOMP's ``advpng``:
+When adding images to the documentation, try to keep them as small as possible.
+Larger images make the site load more slowly on browsers, and may make the site
+inaccessible for people with a slow internet connection.
+
+If you're adding screenshots, make the size of your shot as small as possible.
+If you're uploading large images, consider using an image optimizer in order
+to reduce its size.
+
+For example, for PNG files, use OptiPNG and AdvanceCOMP's ``advpng``:
 
 .. code-block:: console
 

docs/contributing/packages.rst

@@ -0,0 +1,48 @@
+.. _contributing/packages:
+
+=======================
+Environments & Packages
+=======================
+
+TLJH installs packages from different sources during installation.
+This document describes the various sources and how to upgrade
+versions of packages installed.
+
+Python Environments
+===================
+
+TLJH sets up two python environments during installation.
+
+1. **Hub Environment**. JupyterHub, authenticators, spawners, TLJH plugins
+   and the TLJH configuration management code is installed into this
+   environment. A `venv <https://docs.python.org/3/library/venv.html>`_ is used,
+   primarily since conda does not support ARM CPUs and we'd like to support the
+   RaspberryPI someday. Admins generally do not install custom packages
+   in this environment.
+
+2. **User Environment**. Jupyter Notebook, JupyterLab, nteract, kernels,
+   and packages the users wanna use (such as numpy, scipy, etc) are installed
+   here. A `conda <https://conda.io>`_ environment is used here, since
+   a lot of scientific packages are available from Conda. ``pip`` is still
+   used to install Jupyter specific packages, primarily because most notebook
+   extensions are still available only on `PyPI <https://pypi.org>`_.
+   Admins can install packages here for use by all users.
+
+Python package versions
+=======================
+
+In ``installer.py``, most Python packages have a version specified. This
+can be upgraded freely whenever needed. Some of them have version checks
+in ``integration-tests/test_extensions.py``, so those might need
+updating too.
+
+Apt packages
+============
+
+Base operating system packages, including Python itself, are installed
+via ``apt`` from the base Ubuntu repositories. The one exception to this
+is nodejs, which is installed from the `nodesource <https://github.com/nodesource/distributions>`_
+apt repository. The Ubuntu provided version of nodejs is usually too old.
+
+We generally do not pin versions of packages provided by apt, instead
+just using the latest versions provided by Ubuntu.

docs/contributing/plugins.rst

@@ -88,7 +88,7 @@ A hook implementation is a function that has the following characteristics:
    ``tljh.hooks``.
 
 The current list of available hooks and when they are called can be
-seen in ```tljh/hooks.py`` <https://github.com/jupyterhub/the-littlest-jupyterhub/blob/master/tljh/hooks.py>`_
+seen in `tljh/hooks.py <https://github.com/jupyterhub/the-littlest-jupyterhub/blob/master/tljh/hooks.py>`_
 in the source repository.
 
 

docs/doc-requirements.txt

@@ -1 +0,0 @@
-sphinx_copybutton

docs/howto/admin/enable-extensions.rst

@@ -0,0 +1,60 @@
+.. _howto/admin/extensions:
+
+====================================
+Enabling Jupyter Notebook extensions
+====================================
+
+Jupyter contributed notebook 
+`extensions <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/index.html>`_ are
+community-contributed and maintained plug-ins to the Jupyter notebook. These extensions
+serve many purposes, from `pedagogical tools <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/codefolding/readme.html>`_
+to tools for `converting <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/latex_envs/README.html>`_
+and `editing <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/spellchecker/README.html>`_
+notebooks.
+
+Extensions are often added and enabled through the graphical user interface of the notebook.
+However, this interface only makes the extension available to the user, not all users on a 
+hub. Instead, to make contributed extensions available to your users, you will use the command 
+line. This can be completed using the terminal in the JupyterHub (or via SSH-ing into your 
+VM and using this terminal).
+
+.. _tljh_extension_cli:
+
+Enabling extensions via the command line 
+========================================
+
+#. There are `multiple ways <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/install.html>`_
+   to install contributed extensions. For this example, we will use ``pip``.
+
+   .. code-block:: bash
+
+      sudo -E pip install jupyter_contrib_nbextensions
+
+#. Next, add the notebook extension style files to the Jupyter configuration files.
+
+   .. code-block:: bash
+
+      sudo -E jupyter contrib nbextension install --sys-prefix
+
+#. Then, you will enable the extensions you would like to use. The syntax for this is 
+   ``jupyter nbextension enable`` followed by the path to the desired extension's main file. 
+   For example, to enable `scratchpad <https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/scratchpad/README.html>`_,
+   you would type the following: 
+
+   .. code-block:: bash
+
+      sudo -E jupyter nbextension enable scratchpad/main --sys-prefix
+
+#. When this is completed, the enabled extension should be visible in the extension list:
+
+   .. code-block:: bash
+
+      jupyter nbextension list
+
+#. You can also verify the availability of the extension via its user interface in the notebook.
+   For example, spellchecker adds an ABC checkmark icon to the interface.
+
+   .. image:: ../../images/admin/enable-spellcheck.png
+      :alt: spellcheck-interface-changes
+
+

docs/howto/admin/https.rst

@@ -5,31 +5,46 @@ Enable HTTPS
 ============
 
 Every JupyterHub deployment should enable HTTPS!
-HTTPS encrypts traffic so that usernames and passwords and other potentially sensitive bits of information are communicated securely.
-The Littlest JupyterHub supports automatically configuring HTTPS via `Let's Encrypt <https://letsencrypt.org>`_,
-or setting it up :ref:`manually <manual_https>` with your own TLS key and certificate.
-If you don't know how to do that,
-then :ref:`Let's Encrypt <letsencrypt>` is probably the right path for you.
 
+HTTPS encrypts traffic so that usernames, passwords and your data are
+communicated securely. sensitive bits of information are communicated
+securely. The Littlest JupyterHub supports automatically configuring HTTPS
+via `Let's Encrypt <https://letsencrypt.org>`_, or setting it up
+:ref:`manually <howto/admin/https/manual>` with your own TLS key and
+certificate. Unless you have a strong reason to use the manual method,
+you should use the :ref:`Let's Encrypt <howto/admin/https/letsencrypt>`
+method.
 
-.. _letsencrypt:
+.. note::
+
+   You *must* have a domain name set up to point to the IP address on
+   which TLJH is accessible before you can set up HTTPS.
+
+.. _howto/admin/https/letsencrypt:
 
 Automatic HTTPS with Let's Encrypt
 ==================================
 
+.. note::
+
+   If the machine you are running on is not reachable from the internet -
+   for example, if it is a machine internal to your organization that
+   is cut off from the internet - you can not use this method. Please
+   set up a DNS entry and HTTPS :ref:`manually <howto/admin/https/manual>`.
+
 To enable HTTPS via letsencrypt::
 
     sudo tljh-config set https.enabled true
     sudo tljh-config set https.letsencrypt.email you@example.com
     sudo tljh-config add-item https.letsencrypt.domains yourhub.yourdomain.edu
 
-where ``you@example.com`` is your email address and ``yourhub.yourdomain.edu`` is the domain where your hub will be running.
+where ``you@example.com`` is your email address and ``yourhub.yourdomain.edu``
+is the domain where your hub will be running.
 
 Once you have loaded this, your config should look like::
 
     sudo tljh-config show
 
-
 .. sourcecode:: yaml
 
     https:
@@ -43,10 +58,15 @@ Finally, you can reload the proxy to load the new configuration::
 
     sudo tljh-config reload proxy
 
-At this point, the proxy should negotiate with Let's Encrypt to set up a trusted HTTPS certificate for you.
-It may take a moment for the proxy to negotiate with Let's Encrypt to get your certificates, after which you can access your Hub securely at https://yourhub.yourdomain.edu.
+At this point, the proxy should negotiate with Let's Encrypt to set up a
+trusted HTTPS certificate for you. It may take a moment for the proxy to
+negotiate with Let's Encrypt to get your certificates, after which you can
+access your Hub securely at https://yourhub.yourdomain.edu.
 
-.. _manual_https:
+These certificates are valid for 3 months. The proxy will automatically
+renew them for you before they expire.
+
+.. _howto/admin/https/manual:
 
 Manual HTTPS with existing key and certificate
 ==============================================
@@ -77,3 +97,9 @@ Finally, you can reload the proxy to load the new configuration::
     sudo tljh-config reload proxy
 
 and now access your Hub securely at https://yourhub.yourdomain.edu.
+
+Troubleshooting
+===============
+
+If you're having trouble with HTTPS, looking at the :ref:`traefik
+proxy logs <troubleshooting/logs/traefik>` might help.

docs/howto/admin/resize.rst

@@ -0,0 +1,60 @@
+.. _howto/admin/resize:
+
+=================================================
+Resize the resources available to your JupyterHub 
+=================================================
+
+As you are using your JupyterHub, you may need to increase or decrease 
+the amount of resources allocated to your TLJH install. The kinds of resources that can be
+allocated, as well as the process to do so, will depend on the provider / interface for your
+VM. We recommend consulting the installation page for your provider for more information. This
+page covers the steps your should take on your JupyterHub *after* you've reallocated resources on
+the cloud provider of your choice.
+
+Currently there are instructions to resize your resources on the following providers:
+
+* :ref:`Digital Ocean <howto/providers/digitalocean/resize>`.
+
+Once resources have been reallocated, you must tell TLJH to make use of these resources,
+and verify that the resources have become available.
+
+Verifying a Resize 
+==================
+
+#. Once you have resized your server, tell the JupyterHub to make use of 
+   these new resources. To accomplish this, follow the instructions in 
+   :ref:`topic/tljh-config` to set new memory or CPU limits and reload the hub. This can be completed 
+   using the terminal in the JupyterHub (or via SSH-ing into your VM and using this terminal).
+
+#. TLJH configuration options can be verified by viewing the tljh-config output.
+
+   .. code-block:: bash
+
+      sudo tljh-config show
+
+   Double-check that your changes are reflected in the output.
+
+#. **To verify changes to memory**, confirm that it worked by starting
+   a new server (if you had one previously running, click "Control Panel -> Stop My Server" to
+   shut down your active server first), opening a notebook, and checking the value of the 
+   `nbresuse <https://github.com/yuvipanda/nbresuse>`_ extension in the upper-right.
+
+   .. image:: ../../images/nbresuse.png
+      :alt: nbresuse demonstration
+
+#. **To verify changes to CPU**, use the ``nproc`` from a terminal. 
+   This command displays the number of available cores, and should be equal to the 
+   number of cores you selected in your provider's interface.
+
+   .. code-block:: bash
+
+      nproc --all
+
+#. **To verify currently-available disk space**, use the ``df`` command in a terminal. This shows
+   how much disk space is available. The ``-hT`` argument allows us to have this printed in a human readable
+   format, and condenses the output to show one storage volume. Note that currently you cannot
+   change the disk space on a per-user basis.
+
+   .. code-block:: bash
+
+      df -hT /home

docs/howto/admin/resource-estimation.rst

@@ -12,11 +12,13 @@ Memory
 ======
 
 Memory is usually the biggest determinant of server size in most JupyterHub
-installations.
+installations. At minimum, your server must have at least **768MB** of RAM
+for TLJH to install.
 
 .. math::
 
-    Server Memory Recommended = (Maximum concurrent users \times Maximum memory allowed per user) + 128MB
+    Recommended\, Memory =
+    (Max\, concurrent\, users \times Max\, mem\, per\, user) + 128MB
 
 
 The ``128MB`` is overhead for TLJH and related services. **Server Memory Recommended**
@@ -27,7 +29,7 @@ Maximum concurrent users
 ------------------------
 
 Even if your class has 100 students, most of them will not be using the JupyterHub
-actively at an given moment. At 2am on a normal night, maybe you'll have 10 students
+actively at a single given moment. At 2am on a normal night, maybe you'll have 10 students
 using it. At 2am before a final, maybe you'll have 60 students using it. Maybe
 you'll have a lab session with all 100 of your students using it at the same time.
 
@@ -38,7 +40,7 @@ over time. We generally recommend between 40-60% of your total class size to sta
 Maximum memory allowed per user
 -------------------------------
 
-Depending on what kinda work your users are doing, they will use different amounts
+Depending on what kind of work your users are doing, they will use different amounts
 of memory. The easiest way to determine this is to run through a typical user
 workflow yourself, and measure how much memory is used. You can use :ref:`howto/admin/nbresuse`
 to determine how much memory your user is using.
@@ -47,7 +49,7 @@ A good rule of thumb is to take the maximum amount of memory you used during
 your session, and add 20-40% headroom for users to 'play around'. This is the
 maximum amount of memory that should be given to each user.
 
-If users use *more* than this alloted amount of memory, their notebook kernel will restart.
+If users use *more* than this alloted amount of memory, their notebook kernel will *restart*.
 
 CPU
 ===
@@ -58,11 +60,13 @@ stop, unlike with RAM.
 
 .. math::
 
-    Server CPU Recommended = (Maximum concurrent users \times Maximum CPU usage per user) + 0.2
+    Recommended\, CPU = (Max\, concurrent\, users \times Max\, CPU\, usage\, per\, user) + 20\%
 
-The ``0.2`` is overhead for TLJH and related services. **Server CPU Recommended**
-is the amount of CPU the server you acquire should have. We recommend using
+The ``20%`` is overhead for TLJH and related services. This is around 20% of a 
+single modern CPU. This, of course, is just an estimate. We recommend using
 the same process used to estimate Memory required for estimating CPU required.
+You cannot use nbresuse for this, but you should carry out normal workflow and
+investigate the CPU usage on the machine.
 
 Disk space
 ==========
@@ -72,7 +76,7 @@ rather than **maximum concurrent** users.
 
 .. math::
 
-    Server Disk Size Recommended = (Total \times Maximum disk usage per user) + 2GB
+    Recommended\, Disk\, Size = (Total\, users \times Max\, disk\, usage\, per\, user) + 2GB
 
 Resizing your server
 ====================

docs/howto/auth/github.rst

@@ -19,11 +19,11 @@ Step 1: Create a GitHub application
 #. Go to the `GitHub OAuth app creation page <https://github.com/settings/applications/new>`_.
 
    * **Application name**: Choose a descriptive application name (e.g. ``tljh``)
-   * **Homepage URL**: Use the IP address or URL of your JupyterHub. e.g. ``http://<my-tljh-url>```.
+   * **Homepage URL**: Use the IP address or URL of your JupyterHub. e.g. ``http(s)://<my-tljh-url>```.
    * **Application description**: Use any description that you like.
    * **Authorization callback URL**: Insert text with the following form::
 
-          http://<my-tljh-ip-address>/hub/oauth_callback
+          http(s)://<my-tljh-ip-address>/hub/oauth_callback
 
    * When you're done filling in the page, it should look something like this:
 
@@ -55,13 +55,17 @@ For more information on ``tljh-config``, see :ref:`topic/tljh-config`.
    .. image:: ../../images/notebook/new-terminal-button.png
       :alt: New terminal button.
 
-#. Configure the GitHub OAuthenticator to use your client ID and secret with the following commands::
+#. Configure the GitHub OAuthenticator to use your client ID, client secret and callback URL with the following commands::
 
      sudo tljh-config set auth.GitHubOAuthenticator.client_id '<my-tljh-client-id>'
 
    ::
 
      sudo tljh-config set auth.GitHubOAuthenticator.client_secret '<my-tljh-client-secret>'
+   
+   ::
+
+     sudo tljh-config set auth.GitHubOAuthenticator.oauth_callback_url 'http(s)://<my-tljh-ip-address>/hub/oauth_callback'
 
 #. Tell your JupyterHub to *use* the GitHub OAuthenticator for authentication::
 
@@ -71,8 +75,8 @@ For more information on ``tljh-config``, see :ref:`topic/tljh-config`.
 
      sudo tljh-config reload
 
-Confirm that the new authentactor works
-=======================================
+Confirm that the new authenticator works
+========================================
 
 #. **Open an incognito window** in your browser (do not log out until you confirm
    that the new authentication method works!)

docs/howto/auth/google.rst

@@ -0,0 +1,119 @@
+.. _howto/auth/google:
+
+=========================
+Authenticate using Google
+=========================
+
+The **Google Authenticator** lets users log into your JupyterHub using their
+Google user ID / password. To do so, you'll first need to register an
+application with Google, and then provide information about this
+application to your ``tljh`` configuration.
+See `Google's documentation <https://developers.google.com/identity/protocols/OAuth2>`_
+on how to create OAUth 2.0 client credentials.
+
+
+.. note::
+
+   You'll need a Google account in order to complete these steps.
+
+Step 1: Create a Google project
+===============================
+
+Go to `Google Developers Console <https://console.developers.google.com>`_
+and create a new project:
+
+  .. image:: ../../images/auth/google/create_new_project.png
+      :alt: Create a Google project
+
+
+Step 2: Set up a Google OAuth client ID and secret
+==================================================
+
+1. After creating and selecting the project:
+
+  * Go to the credentials menu:
+
+  .. image:: ../../images/auth/google/credentials_button.png
+      :alt: Credentials menu
+
+  * Click "Create credentials" and from the dropdown menu select **"OAuth client ID"**:
+
+  .. image:: ../../images/auth/google/create_credentials.png
+      :alt: Generate credentials
+
+  * You will have to fill a form with:
+     * **Application type**: Choose *Web application*
+     * **Name**: A descriptive name for your OAuth client ID (e.g. ``tljh-client``)
+     * **Authorized JavaScript origins**: Use the IP address or URL of your JupyterHub. e.g. ``http(s)://<my-tljh-url>``.
+     * **Authorized redirect URIs**: Insert text with the following form::
+
+          http(s)://<my-tljh-ip-address>/hub/oauth_callback
+
+     * When you're done filling in the page, it should look something like this (ideally without the red warnings):
+
+      .. image:: ../../images/auth/google/create_oauth_client_id.png
+         :alt: Create a Google OAuth client ID
+
+
+2. Click "Create". You'll be taken to a page with the registered application details.
+3. Copy the **Client ID** and **Client Secret** from the application details
+   page. You will use these later to configure your JupyterHub authenticator.
+
+   .. image:: ../../images/auth/google/client_id_secret.png
+      :alt: Your client ID and secret
+
+.. important::
+
+   If you are using a virtual machine from a cloud provider and
+   **stop the VM**, then when you re-start the VM, the provider will likely assign a **new public
+   IP address** to it. In this case, **you must update your Google application information**
+   with the new IP address.
+
+Configure your JupyterHub to use the Google Oauthenticator
+==========================================================
+
+We'll use the ``tljh-config`` tool to configure your JupyterHub's authentication.
+For more information on ``tljh-config``, see :ref:`topic/tljh-config`.
+
+#. Log in as an administrator account to your JupyterHub.
+#. Open a terminal window.
+
+   .. image:: ../../images/notebook/new-terminal-button.png
+      :alt: New terminal button.
+
+#. Configure the Google OAuthenticator to use your client ID, client secret and callback URL with the following commands::
+
+     sudo tljh-config set auth.GoogleOAuthenticator.client_id '<my-tljh-client-id>'
+
+   ::
+
+     sudo tljh-config set auth.GoogleOAuthenticator.client_secret '<my-tljh-client-secret>'
+   
+   ::
+
+     sudo tljh-config set auth.GoogleOAuthenticator.oauth_callback_url 'http(s)://<my-tljh-ip-address>/hub/oauth_callback'
+
+#. Tell your JupyterHub to *use* the Google OAuthenticator for authentication::
+
+     sudo tljh-config set auth.type oauthenticator.google.GoogleOAuthenticator
+
+#. Restart your JupyterHub so that new users see these changes::
+
+     sudo tljh-config reload
+
+Confirm that the new authenticator works
+========================================
+
+#. **Open an incognito window** in your browser (do not log out until you confirm
+   that the new authentication method works!)
+#. Go to your JupyterHub URL.
+#. You should see a Google login button like below:
+
+   .. image:: ../../images/auth/google/login_button.png
+      :alt: The Google authenticator login button.
+
+#. After you log in with your Google credentials, you should be directed to the
+   Jupyter interface used in this JupyterHub.
+
+#. **If this does not work** you can revert back to the default
+   JupyterHub authenticator by following the steps in :ref:`howto/auth/firstuse`.

docs/howto/auth/nativeauth.rst

@@ -0,0 +1,40 @@
+.. _howto/auth/nativeauth:
+
+==============================================
+Let users sign up with a username and password
+==============================================
+
+The **Native Authenticator** lets users signup for creating a new username 
+and password.
+When they signup, they won't be able to login until they are authorized by an 
+admin. Users that are characterized as admin have to signup as well, but they  
+will be authorized automatically.
+
+
+Enabling the authenticator
+==========================
+
+Enable the authenticator and reload config to apply the configuration:
+
+.. code-block:: bash
+
+   sudo tljh-config set auth.type nativeauthenticator.NativeAuthenticator
+   sudo tljh-config reload
+
+
+Allowing all users to be authorized after signup
+================================================
+
+By default, all users created on signup don't have authorization to login. 
+If you wish to allow **any** user to access
+the JupyterHub just after the signup, run the following command:
+
+.. code-block:: bash
+
+   tljh-config set auth.NativeAuthenticator.open_signup true
+   tljh-config reload
+
+Optional features
+=================
+
+More optional features are available on the `authenticator documentation <https://native-authenticator.readthedocs.io/en/latest/>` 

docs/howto/content/add-data.rst

@@ -83,7 +83,7 @@ time. You can download it from your browser `at this link <https://swcarpentry.g
 
       sudo apt install unzip
 
-#. Finally, unzip the the file:
+#. Finally, unzip the file:
 
    unzip python-novice-gapminder-data.zip
 

docs/howto/content/share-data.rst

@@ -67,7 +67,7 @@ follow these steps:
 
    .. code-block:: bash
 
-      sudo ln -s /src/data/my_shared_data_folder my_shared_data_folder
+      sudo ln -s /srv/data/my_shared_data_folder my_shared_data_folder
 
 #. **Confirm that this worked** by logging in as a new user. You can do this
    by opening a new "incognito" browser window and accessing your JupyterHub.

docs/howto/env/server-resources.rst

@@ -0,0 +1,10 @@
+.. _howto/env/server-resources:
+
+======================================
+Configure resources available to users
+======================================
+
+To configure the resources that are available to your users (such as RAM, CPU
+and Disk Space), see the section :ref:`tljh-set-user-limits`. For information
+on **resizing** the environment available to users *after* you've created your
+JupyterHub, see :ref:`howto/admin/resize`.

docs/howto/providers/azure.rst

@@ -0,0 +1,28 @@
+.. _howto/providers/azure:
+
+==================================================
+Perform common Microsoft Azure configuration tasks
+==================================================
+
+This page lists various common tasks you can perform on your
+Microsoft Azure virtual machine.
+
+.. _howto/providers/azure/resize:
+
+Deleting or stopping your virtual machine
+===========================================
+
+After you have finished using your TLJH you might wanto to either Stop or completely delete the Virtual Machine to avoid incurring in subsequent costs. 
+The difference between these two approaches is that **Stop** will keep the VM resources but will effectively stop any compute / runtime activities. 
+If you choose to delete the VM then all the resources associated with it will be wiped out.
+
+To do either of this:
+
+* Go to "Virtual Machines"
+* Click on your machine name
+* Click on "Stop" to stop the machine temporarily, or "Delete" to delete it permanently.
+
+    .. image:: ../../images/providers/azure/delete-vm.png
+        :alt: Delete vm
+
+.. note:: It is important to mention that even if you stop the machine you will still be charged for the use of the data disk.

docs/howto/providers/digitalocean.rst

@@ -0,0 +1,43 @@
+.. _howto/providers/digitalocean:
+
+================================================
+Perform common Digital Ocean configuration tasks
+================================================
+
+This page lists various common tasks you can perform on your
+Digital Ocean virtual machine.
+
+.. _howto/providers/digitalocean/resize:
+
+Resizing your droplet
+=====================
+
+As you use your JupyterHub, you may find that you need more memory, 
+disk space, or CPUs. Digital Ocean servers can be resized in the 
+"Resize Droplet" panel. These instructions take you through the process.
+
+#. First, click on the name of your newly-created
+   Droplet to enter its configuration page.
+
+#. Next, **turn off your Droplet**. This allows DigitalOcean to make
+   modifications to your VM. This will shut down your JupyterHub (temporarily).
+
+   .. image:: ../../images/providers/digitalocean/power-off.png
+      :alt: Power off your Droplet
+      :width: 200px
+
+#. Once your Droplet has been turned off, click "Resize",
+   which will take you to a menu with options to resize your VM.
+
+   .. image:: ../../images/providers/digitalocean/resize-droplet.png
+      :alt: Resize panel of digital ocean
+
+#. Decide what kinds of resources you'd like to resize, then click on a new VM
+   type in the list below. Finally, click "Resize". This may take a few moments!
+
+#. Once your Droplet is resized, **turn your Droplet back on**. This makes your JupyterHub
+   available to the world once again. This will take a few moments to complete.
+
+Now that you've resized your Droplet, you may want to change the resources available
+to your users. Further information on making more resources available to
+users and verifying resource availability can be found in :ref:`howto/admin/resize`. 

docs/index.rst

@@ -6,19 +6,20 @@ A simple `JupyterHub <https://github.com/jupyterhub/jupyterhub>`_ distribution f
 a small (0-100) number of users on a single server. We recommend reading
 :ref:`topic/whentouse` to determine if this is the right tool for you.
 
+
 Development Status
 ==================
 
-This project is currently in **alpha** state. Most things work, but we might
-still make breaking changes that have no clear upgrade pathway. We are targeting
-a v0.1 release sometime in mid-August 2018. Follow `this milestone <https://github.com/jupyterhub/the-littlest-jupyterhub/milestone/1>`_
-to see progress towards the release!
+This project is currently in **beta** state. Folks have been using installations
+of TLJH for more than a year now to great success. While we try hard not to, we
+might still make breaking changes that have no clear upgrade pathway.
 
 Installation
 ============
 
 The Littlest JupyterHub (TLJH) can run on any server that is running at least
-Ubuntu 18.04. We have a bunch of tutorials to get you started.
+**Ubuntu 18.04**. Earlier versions of Ubuntu are not supported.
+We have a bunch of tutorials to get you started.
 
 - Tutorials to create a new server from scratch on a cloud provider & run TLJH
   on it. These are **recommended** if you do not have much experience setting up
@@ -26,10 +27,14 @@ Ubuntu 18.04. We have a bunch of tutorials to get you started.
 
   .. toctree::
      :titlesonly:
+     :caption: Installation
 
      install/digitalocean
+     install/ovh
      install/jetstream
      install/google
+     install/amazon
+     install/azure
      install/custom-server
 
 Once you are ready to run your server for real,
@@ -45,6 +50,7 @@ Content and Data
 
 .. toctree::
    :titlesonly:
+   :caption: Content and data
 
    howto/content/nbgitpuller
    howto/content/add-data
@@ -55,9 +61,11 @@ The user environment
 
 .. toctree::
    :titlesonly:
+   :caption: The user environment
 
    howto/env/user-environment
    howto/env/notebook-interfaces
+   howto/env/server-resources
 
 Authentication
 --------------
@@ -68,21 +76,37 @@ with your JupyterHub. For more information on Authentication, see
 
 .. toctree::
    :titlesonly:
+   :caption: Authentication
 
    howto/auth/dummy
    howto/auth/github
+   howto/auth/google
    howto/auth/firstuse
+   howto/auth/nativeauth
 
 Administration and security
 ---------------------------
 
 .. toctree::
    :titlesonly:
+   :caption: Administration and security
 
    howto/admin/admin-users
    howto/admin/resource-estimation
+   howto/admin/resize
    howto/admin/nbresuse
    howto/admin/https
+   howto/admin/enable-extensions
+
+Cloud provider configuration
+----------------------------
+
+.. toctree::
+   :titlesonly:
+   :caption: Cloud provider configuration
+
+   howto/providers/digitalocean
+   howto/providers/azure
 
 Topic Guides
 ============
@@ -91,6 +115,7 @@ Topic guides provide in-depth explanations of specific topics.
 
 .. toctree::
    :titlesonly:
+   :caption: Topic guides
 
    topic/whentouse
    topic/requirements
@@ -100,6 +125,7 @@ Topic guides provide in-depth explanations of specific topics.
    topic/tljh-config
    topic/authenticator-configuration
    topic/escape-hatch
+   topic/idle-culler
 
 
 Troubleshooting
@@ -110,6 +136,7 @@ guides help you find what is broken & hopefully fix it.
 
 .. toctree::
    :titlesonly:
+   :caption: Troubleshooting
 
    troubleshooting/logs
 
@@ -122,6 +149,8 @@ here to better support your favorite provider!
    :titlesonly:
 
    troubleshooting/providers/google
+   troubleshooting/providers/amazon
+   troubleshooting/providers/custom
 
 Contributing
 ============
@@ -132,9 +161,11 @@ to people contributing in various ways.
 
 .. toctree::
    :titlesonly:
+   :caption: Contributing
 
    contributing/docs
    contributing/code-review
    contributing/dev-setup
    contributing/tests
-   contributing/plugins
+   contributing/plugins
+   contributing/packages

docs/install/amazon.rst

@@ -0,0 +1,277 @@
+.. _install/amazon:
+
+=================================
+Installing on Amazon Web Services
+=================================
+
+Goal
+====
+
+To have a JupyterHub with admin users and a user environment with conda / pip packages.
+
+Prerequisites
+=============
+
+#. An Amazon Web Services account.
+
+   The `AWS free tier <https://aws.amazon.com/free/>`_ is fully
+   capable of running a minimal littlest Jupyterhub for testing purposes. 
+
+   If asked to choose a default region, choose the one closest to the majority 
+   of your users.
+
+Step 1: Installing The Littlest JupyterHub
+==========================================
+
+Let's create the server on which we can run JupyterHub.
+
+#. Go to `Amazon Web Services <https://aws.amazon.com/>`_ and click the gold 
+   button 'Sign In to the Console' in the upper right. Log in with your Amazon Web 
+   Services account.
+   
+   If you need to adjust your region from your default, there is a drop-down 
+   menu between your name and the **Support** menu on the far right of the dark 
+   navigation bar across the top of the window. Adjust the region to match the
+   closest one to the majority of your users.
+
+#. On the screen listing all the availabe services, pick **EC2** under **Compute** 
+   on the left side at the top of the first column. 
+   
+   .. image:: ../images/providers/amazon/compute_services.png
+      :alt: Select EC2
+   
+   This will take you to the **EC2 Management Console**.
+
+#. From the navigation menu listing on the far left side of the **EC2 Management 
+   Console**, choose **Instances** under the light gray **INSTANCES** sub-heading.
+
+   .. image:: ../images/providers/amazon/instances_from_console.png
+      :alt: Select Instances from console
+
+#. In the main window of the **EC2 Management Console**, towards the top left, 
+   click on the bright blue **Launch Instance** button.
+   
+   .. image:: ../images/providers/amazon/launch_instance_button.png
+      :alt: Click launch instance
+   
+   This will start the 'launch instance wizard' process.  This lets you customize
+   the kind of server you want, the resources it will have and its name.
+   
+   
+#. On the page **Step 1: Choose an Amazon Machine Image (AMI)** you are going 
+   to pick the base image your remote server will have. The view will 
+   default to the 'Quick-start' tab selected and just a few down the page, select 
+   **Ubuntu Server 18.04 LTS (HVM), SSD Volume Type - ami-XXXXXXXXXXXXXXXXX**.
+   
+   .. image:: ../images/providers/amazon/select_ubuntu_18.png
+      :alt: Click Ubuntu server 18.04
+   
+   The `ami` alpha-numeric at the end references the specific Amazon machine 
+   image, ignore this as Amazon updates them routinely. The 
+   **Ubuntu Server 18.04 LTS (HVM)** is the important part.
+   
+   
+#. After selecting the AMI, you'll be at **Step 2: Choose an Instance Type**.
+   
+   There will be a long listing of the types and numbers of CPUs that Amazon 
+   offers. Select the one you want and then select the button 
+   `Next: Configure Instance Details` in the lower right corner.  
+   
+   Check out our guide on How To :ref:`howto/admin/resource-estimation` to help pick
+   how much Memory / CPU your server needs. You need to have at least **768MB** of
+   RAM.
+   
+   You may wish to consult the listing `here <https://www.ec2instances.info/>`_ 
+   because it shows cost per hour. The **On Demand** price is the pertinent cost.
+   
+   (For reference, a minimal hub that worked for developing this tutorial used a
+   **t2.micro** tier, which is free for Amazon users the first year they sign
+   up. Two users were able to concurrently utilize this development hub without issue.)
+   
+   ``GPU graphics`` and ``GPU compute`` products are also available around half way down the page
+   
+#. Under **Step 3: Configure Instance Details**, scroll to the bottom of the page 
+   and toggle the arrow next to **Advanced Details**. Scroll down to 'User data'. Copy 
+   the text below, and paste it into the **User data** text box. Replace
+   ``<admin-user-name>`` with the name of the first **admin user** for this
+   JupyterHub. This admin user can log in after the JupyterHub is set up, and
+   configure it. **Remember to add your username**!
+
+   .. code-block:: bash
+
+       #!/bin/bash
+       curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
+         | sudo python3 - \
+           --admin <admin-user-name>
+
+   .. image:: ../images/providers/amazon/script_in_user_data.png
+      :alt: Install JupyterHub with the script in the User data textbox
+
+   .. note::
+
+      See :ref:`topic/installer-actions` for a detailed description and 
+      :ref:`topic/customizing-installer` for other options that can be used.
+   
+#. Under **Step 4: Add Storage**, you can change the **size** and **type of your 
+   disk by adjusting the value in **Size (GiB)** and selecting **Volume Type**.
+   
+   .. image:: ../images/providers/amazon/change_size_type.png
+      :alt: Selecting disk size and type
+   
+   Check out :ref:`howto/admin/resource-estimation` to help pick
+   how much Disk space your server needs.
+   
+   Hover over the encircled `i` next to **Volume Type** for an explanation of
+   each. Leaving the default as is is fine. `General Purpose SSD (gp2)` is 
+   recommended for most workloads. With  `Provisioned IOPS SSD (io1)` being the 
+   highest-performance SSD volume. Magnetic (standard) is a previous generation 
+   volume and not suited for a hub for multi-users.
+   
+   When finished, click **Next: Add Tags** in the bottom right corner.
+   
+#. Under **Step 5: Add Tags**, click **Add Tag** and enter **Name** under the 
+   **Key** field. In the **Value** field in the **Name** row, give your new 
+   server a memorable name that identifies what purpose this JupyterHub will be 
+   used for.
+   
+   .. image:: ../images/providers/amazon/name_hub.png
+      :alt: Use tags to name the hub.
+
+#. Under **Step 6: Configure Security Group**, you'll set the firewall rules 
+   that control the traffic for your instance. Specifically you'll want to add 
+   rules to allow both **HTTP Traffic** and **HTTPS Traffic**. For 
+   advanced troubleshooting, it will be helpful to set rules so you can use 
+   SSH to connect (port 22). 
+   
+   If you have never used your Amazon account before, you'll have to select 
+   **Create a new security group**. You should give it a disitnguishing name 
+   under **Security group name**
+   such as `ssh_web` for future reference. If you have, one from before you can 
+   select it and adjust it to have the rules you need, if you prefer.
+   
+   The rules will default to include `SSH`. Leave that there, and then click on 
+   the **Add Rule** button. Under **Type** for the new rule, change the field 
+   to **HTTP**. The other boxes will get filled in appropritely. Again, click on 
+   the **Add Rule** button. This time under **Type** for the new rule, change 
+   the field to **HTTPS**.
+   
+   The warning is there to remind you this opens things up to some degree but 
+   this is necessary in order to let your users connect. However, this warning 
+   is a good reminder that you should monitor your server to insure it is 
+   available for users who may need it.
+   
+   .. image:: ../images/providers/amazon/set_security_groups.png
+      :alt: Allow HTTP & HTTPS traffic to your server
+   
+#. When the security rules are set, click on the blue button in the bottom 
+   right **Review and Launch**. This will give you a chance to review things 
+   because very soon you'll be launching and start paying for any resources you 
+   use.
+   
+   Note that you'll see two HTTP listings and two HTTPS listings under 
+   **Security Groups** even though you only made one for each. This is normal &
+   necessary to match both IPv4 & IPv6 types of IP addresses.
+   
+   When you are happy, press the blue **Launch** button in the bottom right 
+   corner to nearly conclude your journey through the instance launch wizard.
+   
+   .. image:: ../images/providers/amazon/finally_launch.png
+      :alt: Launch your server
+   
+#. In the dialog box that pops up as the last step before launching is 
+   triggered, you need to choose what to do about an identifying key pair and 
+   acknowledge your choice in order to proceed. If you already have a key pair you 
+   can select to associate it with this instance, otherwise you need to 
+   **Create a new key pair**. Choosing to `Proceed without a key pair` is not 
+   recommended as you'll have no way to access your server via SSH if anything
+   goes wrong with the Jupyterhub and have no way to recover files via download.
+   
+   Download and keep the key pair file unless you are associating one you already 
+   have.
+   
+   .. image:: ../images/providers/amazon/create_key_pair.png
+      :alt: Associate key pair
+   
+#. With the key pair associated, click the **Launch instances** button to 
+   start creating the server that'll run TLJH.
+   
+   .. image:: ../images/providers/amazon/launch_now.png
+      :alt: Trigger actual launch
+   
+   
+#. Following the launch initiation, you'll be taken to a **Launch Status** 
+   notification screen. You can see more information about the details if you 
+   click on the alphanumeric link to the launching instance following the text, 
+   "`The following instance launches have been initiated:`".
+   
+   .. image:: ../images/providers/amazon/launch_status_screen.png
+      :alt: Launch status notice
+   
+#. That link will take you back to the **EC2 Management Console** with settings
+   that will limit the view in the console to just that instance. (Delete the 
+   filter in the search bar if you want to see any other instances you may 
+   have.) At first the server will be starting up, and then when the 
+   **Instance state** is green the server is running.
+   
+   .. image:: ../images/providers/amazon/running_server.png
+      :alt: Server is running.
+   
+   If you already have instances running in your account, the screen will look
+   different if you disable that filter. But you want to pay attention to the 
+   row with the name of the server you made.
+   
+#. In a few seconds your server will be created, and you can see the 
+   **Public IP** used to access it in the panel at the bottom of the console. 
+   If it isn't displayed, click on the row for that instance in the console. It 
+   will look like a pattern similar to **12.30.230.127**.
+   
+   .. image:: ../images/providers/amazon/public_ip.png
+      :alt: public IP
+   
+#. The Littlest JupyterHub is now installing in the background on your new 
+   server. It takes around 10 minutes for this installation to complete.
+
+#. Check if the installation is complete by copying the **Public IP**
+   of your server, and trying to access it from within a browser. If it has been 
+   10 minutes, paste the public IP into the URL bar of your browser and hit 
+   return to try to connect. 
+   
+   Accessing the JupyterHub will fail until the installation is complete,
+   so be patient.  The next step below this one shows the login window you are 
+   expecting to see when trying the URL and things work. 
+   While waiting until the appropriate time to try, another way to check if 
+   things are churning away, is to open the **System Log**. To do this, go to 
+   the **EC2 Management Console** & highlight the instance by clicking on that 
+   row and then right-click **Instance Settings** > **Get System Log**.
+   
+   .. image:: ../images/providers/amazon/get_system_log.png
+      :alt: Getting system log.
+   
+#. When the Jupyterhub creation process finishes and the hub is ready to show 
+   the login, the **System Log** should look similar to the image below. Scroll to 
+   the bottom of your output from the previous step.
+   Note the line **Starting TLJH installer**, you may also see **Started jupyterhub.service**
+
+   .. image:: ../images/providers/amazon/completed_system_log.png
+      :alt: Completed system log
+   
+#. When the installation is complete, it should give you a JupyterHub login page.
+   
+   .. image:: ../images/first-login.png
+      :alt: JupyterHub log-in page
+   
+#. Login using the **admin user name** you used in step 7, and a password. Use a
+   strong password & note it down somewhere, since this will be the password for
+   the admin user account from now on.
+   
+#. Congratulations, you have a running working JupyterHub!
+
+Step 2: Adding more users
+==========================
+
+.. include:: add_users.txt
+
+Step 3: Install conda / pip packages for all users
+==================================================
+
+.. include:: add_packages.txt

docs/install/azure.rst

@@ -0,0 +1,175 @@
+.. _install/azure:
+
+====================
+Installing on Azure
+====================
+
+Goal
+====
+
+By the end of this tutorial, you should have a JupyterHub with some admin
+users and a user environment with packages you want to be installed running on
+`Microsoft Azure <https://azure.microsoft.com>`_.
+
+Prerequisites
+==============
+
+* A Microsoft Azure account. 
+
+* To get started you can get a free account which  includes 150 dollars worth of Azure credits (`get a free account here <https://azure.microsoft.com/en-us/free//?wt.mc_id=TLJH-github-taallard>`_) 
+
+These instructions cover how to set up a Virtual Machine
+on Microsoft Azure. For subsequent information about creating
+your JupyterHub and configuring it, see `The Littlest JupyterHub guide <https://the-littlest-jupyterhub.readthedocs.io/en/latest/>`_.
+
+
+Step 1: Installing The Littlest JupyterHub
+==========================================
+
+We will start by creating the Virtual Machine in which we can run TLJH (The Littlest JupyterHub).
+
+#. Go to `Azure portal <https://portal.azure.com/>`_  and login with your Azure account.
+#. Expand the left-hand panel, find the Virtual Machines tab and click on it.
+
+.. image:: ../images/providers/azure/azure-vms.png
+    :alt: Virtual machines on Azure portal
+
+#. Click **+ add** to create a new Virtual Machine
+
+    .. image:: ../images/providers/azure/add-vm.png
+        :alt: Add a new virtual machine
+
+#. Select **Create VM from Marketplace** in the next sreen. This will display a new screen with all the optiond for Virtual Machines in Azure.
+    .. image:: ../images/providers/azure/create-vm.png
+        :alt: Create VM from the marketplace
+
+#. **Choose an Ubuntu server for your VM**: 
+    * Click `Ubuntu Server 18.04 LTS`
+    * Make sure `Resource Manager` is selected in the next screen and click **Create**
+
+    .. image:: ../images/providers/azure/ubuntu-vm.png
+        :alt: Ubuntu VM
+
+#. Customise the Virtual Machine basics:
+    * **Subscription**. Choose the "Free Trial" if this is what you're using. Otherwise, choose a different plan. This is the billing account that will be charged.
+    * **Resource group**. Resource groups let you bundle components that you request from Azure. If you already have one you'd like to use it select that resource.
+    * **Name**. Use a descriptive name for your virtual machine (note that you cannot use spaces or special characters).
+    * **Region**. Choose a location near where you expect your users to be located.
+    * **Availability options**. Choose "No infrastructure redundancy required".
+    * **Image**. Make sure "Ubuntu Server 18.04 LTS" is selected (from the previous step).
+    * **Authentication type**. Change authentication type to "password".
+    * **Username**. Choose a memorable username, this will be your "root" user and you'll need it later on.
+    * **Password**. Type in a password, this will be used later for admin access so make sure it is something memorable.
+
+    .. image:: ../images/providers/azure/password-vm.png
+            :alt: Add password to VM
+
+    * **Login with Azure Active Directory**. Choose "Off" (usually the default)
+    * **Inbound port rules**. Leave the defaults for now and we will update these later on in the Network configuration step.
+
+#. Before clicking on "Next" we need to select the RAM size for the image.
+    * For this we need to make sure we have enough RAM to accommodate your users. For example, if each user needs 2GB of RAM, and you have 10 total users, you need at least 20GB of RAM on the machine. It's also good to have a few GB of "buffer" RAM beyond what you think you'll need.
+    * Click on **Change size** (see image below) 
+
+    .. image:: ../images/providers/azure/size-vm.png
+            :alt: Choose vm size 
+
+    .. note:: For more information about estimating memory, CPU and disk needs check `The memory section in the TLJH documentation <https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html>`_ 
+
+    * Select a suitable image (to check available images and prices in your region `click on this link <https://azuremarketplace.microsoft.com/en-gb/marketplace/apps/Canonical.UbuntuServer?tab=PlansAndPrice/?wt.mc_id=TLJH-github-taallard>`_.    
+    
+#. Disks (Storage):
+    * **Disk options**: slect the OS disk type there are options for SDD and HDD. **SSD persistent disk** gives you a faster but more expensive disk than HDD. 
+    * **Data disk**. Click on create and attach a new disk. Select an appropriate type and size and click ok.
+    * Click "Next"
+
+    .. image:: ../images/providers/azure/disk-vm.png
+            :alt: Choose disk size  
+
+#. Networking
+    * **Virtual network**. Leave the default values selected.
+    * **Subnet**. Leave the default values selected.
+    * **Public IP address**.Leave the default values selected. This will make your server accessible from a browser.
+    * **Network Security Group**. Choose "Basic"
+    * **Public inbound ports**. Check **HTTP**, **HTTPS**, and **SSH**.
+
+    .. image:: ../images/providers/azure/networking-vm.png
+            :alt: Choose networking ports 
+
+#. Management
+    * Monitoring
+        * **Boot diagnostics**. Choose "On".
+        * **OS guest diagnostics**. Choose "Off".
+        * **Diagnostics storage account**. Leave as the default.
+    * Auto-Shutdown
+        * **Enable auto-shutdown**. Choose "Off".
+    * Backup
+        * **Backup**. Choose "Off".
+    * System assigned managed identity Select "Off"
+
+    .. image:: ../images/providers/azure/backup-vm.png
+                :alt: Choose VM Backup
+
+#. Advanced settings
+    * **Extensions**. Make sure there are no extensions listed
+    * **Cloud init**. We are going to use this section to install TLJH directly into our Virtual Machine. 
+      Copy the code snippet below:
+
+      .. code:: bash
+
+         #!/bin/bash
+         curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
+           | sudo python3 - \
+             --admin <admin-user-name>
+
+      where the ``username`` is the root username you chose for your Virtual Machine.
+
+      .. image:: ../images/providers/azure/cloudinit-vm.png
+                 :alt: Install TLJH
+
+    .. note::
+
+        See :ref:`topic/installer-actions` if you want to understand exactly what the installer is doing.
+        :ref:`topic/customizing-installer` documents other options that can be passed to the installer.
+
+#. Check the summary and confirm the creation of your Virtual Machine.
+
+#. Check that the creation of your Virtual Machine worked
+    * Wait for the virtual machine to be created. This might take about 5-10 minutes.
+    * After completion, you should see a similar screen to the one below:
+
+    .. image:: ../images/providers/azure/deployed-vm.png
+        :alt: Deployed VM
+
+#. Note that the Littlest JupyterHub should be installing in the background on your new server.
+    It takes around 5-10 minutes for this installation to complete.
+
+#. Click on the **Go to resource button**
+    .. image:: ../images/providers/azure/goto-vm.png
+        :alt: Go to VM
+
+#. Check if the installation is complete by **copying** the **Public IP address** of your virtual machine, and trying to access it with a browser. 
+
+    .. image:: ../images/providers/azure/ip-vm.png
+        :alt: Public IP address
+
+    Note that accessing the JupyterHub will fail until the installation is complete, so be patient.
+
+#. When the installation is complete, it should give you a JupyterHub login page.
+
+    .. image:: ../images/first-login.png
+        :alt: JupyterHub log-in page
+
+#. Login using the **admin user name** you used in step 6, and a password. Use a strong password & note it down somewhere, since this will be the password for the admin user account from now on.
+
+#. Congratulations, you have a running working JupyterHub! 🎉
+
+Step 2: Adding more users
+==========================
+
+.. include:: add_users.txt
+
+Step 3: Install conda / pip packages for all users
+==================================================
+
+.. include:: add_packages.txt

docs/install/custom-server.rst

@@ -4,11 +4,21 @@
 Installing on your own server
 =============================
 
+
+Follow this guide if your cloud provider doesn't have a direct tutorial, or
+you are setting this up on a bare metal server.
+
+.. warning::
+
+   Do **not** install TLJH directly on your laptop or personal computer!
+   It will most likely open up exploitable security holes when run directly
+   on your personal computer.
+
 .. note::
 
-    You should use this if your cloud provider does not already have a direct tutorial,
-    or if you have experience setting up servers.
-
+   Running TLJH *inside* a docker container is not supported, since we depend
+   on systemd. If you want to run TLJH locally for development, see
+   :ref:`contributing/dev-setup`.
 
 Goal
 ====
@@ -22,8 +32,12 @@ Pre-requisites
 
 #. Some familiarity with the command line.
 #. A server running Ubuntu 18.04 where you have root access.
+#. At least **768MB** of RAM on your server.
 #. Ability to ``ssh`` into the server & run commands from the prompt.
-#. A **public IP** where the server can be accessed from the internet.
+#. A **IP address** where the server can be reached from the browsers of your target audience.
+
+If you run into issues, look at the specific :ref:`troubleshooting guide <troubleshooting/providers/custom>`
+for custom server installations.
 
 Step 1: Installing The Littlest JupyterHub
 ==========================================
@@ -31,21 +45,21 @@ Step 1: Installing The Littlest JupyterHub
 #. Using a terminal program, SSH into your server. This should give you a prompt where you can
    type commands.
 
-#. Make sure you have ``Python3``, ``curl`` and ``git``  installed. On latest Ubuntu you can get all of these with:
+#. Make sure you have ``python3``, ``curl`` and ``git`` installed.
 
-.. code::
+   .. code::
 
-   apt-get install python3 git curl
+      sudo apt install python3 git curl
 
 #. Copy the text below, and paste it into the terminal. Replace
    ``<admin-user-name>`` with the name of the first **admin user** for this
-   JupyterHub. Choose any name you like (don't forget to replace the brackets!).
+   JupyterHub. Choose any name you like (don't forget to remove the brackets!).
    This admin user can log in after the JupyterHub is set up, and
    can configure it to their needs. **Remember to add your username**!
 
    .. code-block:: bash
 
-      curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo python3 - --admin <admin-user-name>
+      curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo -E python3 - --admin <admin-user-name>
 
    .. note::
 

docs/install/digitalocean.rst

@@ -116,4 +116,4 @@ Step 2: Adding more users
 Step 3: Install conda / pip packages for all users
 ==================================================
 
-.. include:: add_packages.txt
+.. include:: add_packages.txt

docs/install/google.rst

@@ -69,7 +69,7 @@ Let's create the server on which we can run JupyterHub.
 #. For **Zone**, pick any of the options. Leaving the default as is is fine.
 
 #. Under **Machine** type, select the amount of CPU / RAM / GPU you want for your
-   server.
+   server. You need at least **768MB** of RAM.
 
    You can select a preset combination in the default **basic view**.
 
@@ -136,7 +136,7 @@ Let's create the server on which we can run JupyterHub.
 
    This displays a lot of advanced options, but we'll be only using one of them.
 
-#. Copy the text below, and paste it into the **Starup script** text box. Replace
+#. Copy the text below, and paste it into the **Startup script** text box. Replace
    ``<admin-user-name>`` with the name of the first **admin user** for this
    JupyterHub. This admin user can log in after the JupyterHub is set up, and
    can configure it to their needs. **Remember to add your username**!

docs/install/jetstream.rst

@@ -53,6 +53,7 @@ Let's create the server on which we can run JupyterHub.
 
    #. Give your server a descriptive **Instance Name**.
    #. Select an appropriate **Instance Size**. We suggest m1.medium or larger.
+      Make sure your instance has at least **768MB** of RAM.
 
       Check out our guide on How To :ref:`howto/admin/resource-estimation` to help pick
       how much Memory, CPU & disk space your server needs.

docs/install/ovh.rst

@@ -0,0 +1,127 @@
+.. _install/ovh:
+
+=================
+Installing on OVH
+=================
+
+Goal
+====
+
+By the end of this tutorial, you should have a JupyterHub with some admin
+users and a user environment with packages you want installed running on
+`OVH <https://www.ovh.com>`_.
+
+Pre-requisites
+==============
+
+#. An OVH account.
+
+Step 1: Installing The Littlest JupyterHub
+==========================================
+
+Let's create the server on which we can run JupyterHub.
+
+#. Log in to the `OVH Control Panel <https://www.ovh.com/auth/>`_.
+
+#. Click the **Public Cloud** button in the navigation bar.
+
+   .. image:: ../images/providers/ovh/public-cloud.png
+      :alt: Public Cloud entry in the navigation bar
+
+#. If you don't have an OVH Stack, you can create one by clicking on the following button:
+
+   .. image:: ../images/providers/ovh/create-ovh-stack.png
+      :alt: Button to create an OVH stack
+      
+#. Select a name for the project:
+
+   .. image:: ../images/providers/ovh/project-name.png
+      :alt: Select a name for the project
+
+#. If you don't have a payment method yet, select one and click on "Create my project":
+
+   .. image:: ../images/providers/ovh/payment.png
+      :alt: Select a payment method
+
+#. Using the **Public Cloud interface**, click on **Create an instance**:
+
+   .. image:: ../images/providers/ovh/create-instance.png
+      :alt: Create a new instance
+      
+#. **Select a model** for the instance. A good start is the **S1-4** model under **Shared resources** which comes with 4GB RAM, 1 vCores and 20GB SSD.
+      
+#. **Select a region**.
+
+#. Select **Ubuntu 18.04** as the image:
+
+   .. image:: ../images/providers/ovh/distribution.png
+      :alt: Select Ubuntu 18.04 as the image
+
+#. OVH requires setting an SSH key to be able to connect to the instance.
+   You can create a new SSH by following
+   `these instructions <https://help.github.com/en/enterprise/2.16/user/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent>`_.
+   Be sure to copy the content of the ``~/.ssh/id_rsa.pub`` file, which corresponds to the **public part** of the SSH key.
+
+#. Select **Configure your instance**, and select a name for the instance.
+   Under **Post-installation script**, copy the text below and paste it in the text box.
+   Replace ``<admin-user-name>`` with the name of the first **admin user** for this
+   JupyterHub. This admin user can log in after the JupyterHub is set up, and
+   can configure it to their needs. **Remember to add your username**!
+
+   .. code-block:: bash
+
+      #!/bin/bash
+      curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
+        | sudo python3 - \
+          --admin <admin-user-name>
+
+   .. note::
+
+      See :ref:`topic/installer-actions` if you want to understand exactly what the installer is doing.
+      :ref:`topic/customizing-installer` documents other options that can be passed to the installer.
+
+
+   .. image:: ../images/providers/ovh/configuration.png
+      :alt: Add post-installation script
+
+#. Select a billing period: monthly or hourly.
+
+#. Click the **Create an instance** button! You will be taken to a different screen,
+   where you can see progress of your server being created.
+
+   .. image:: ../images/providers/ovh/create-instance.png
+      :alt: Select suitable hostname for your server
+
+#. In a few seconds your server will be created, and you can see the **public IP**
+   used to access it.
+
+   .. image:: ../images/providers/ovh/public-ip.png
+      :alt: Server finished creating, public IP available
+
+#. The Littlest JupyterHub is now installing in the background on your new server.
+   It takes around 5-10 minutes for this installation to complete.
+
+#. Check if the installation is complete by copying the **public ip**
+   of your server, and trying to access it with a browser. This will fail until
+   the installation is complete, so be patient.
+
+#. When the installation is complete, it should give you a JupyterHub login page.
+
+   .. image:: ../images/first-login.png
+      :alt: JupyterHub log-in page
+
+#. Login using the **admin user name** you used in step 6, and a password. Use a
+   strong password & note it down somewhere, since this will be the password for
+   the admin user account from now on.
+
+#. Congratulations, you have a running working JupyterHub!
+
+Step 2: Adding more users
+==========================
+
+.. include:: add_users.txt
+
+Step 3: Install conda / pip packages for all users
+==================================================
+
+.. include:: add_packages.txt

docs/requirements.txt

@@ -1,2 +1,4 @@
 sphinx>=1.4, !=1.5.4
 sphinx_copybutton
+alabaster
+alabaster_jupyterhub

docs/topic/authenticator-configuration.rst

@@ -16,6 +16,9 @@ can be used with TLJH. A number of them ship by default with TLJH:
    available.
 #. `FirstUseAuthenticator <https://github.com/yuvipanda/jupyterhub-firstuseauthenticator>`_ - Users set
    their password when they log in for the first time. Default authenticator used in TLJH.
+#. `TmpAuthenticator <https://github.com/jupyterhub/tmpauthenticator>`_ - Opens the JupyterHub to the
+   world, makes a new user every time someone logs in.
+#. `NativeAuthenticator <https://native-authenticator.readthedocs.io/en/latest/>`_ - Allow users to signup, add password security verification and block users after failed attempts oflogin. 
 
 We try to have specific how-to guides & tutorials for common authenticators. Since we can not cover
 everything, this guide shows you how to use any authenticator you want with JupyterHub by following
@@ -47,7 +50,7 @@ to some value, you can do that with the following command:
 
 .. code-block:: bash
 
-   sudo tljh-config set auth.LDAPAuthenticator.server_address = 'my-ldap-server'
+   sudo tljh-config set auth.LDAPAuthenticator.server_address 'my-ldap-server'
 
 Most authenticators require you set multiple configuration options before you can
 enable them. Read the authenticator's documentation carefully for more information.

docs/topic/customizing-installer.rst

@@ -63,12 +63,12 @@ Installing TLJH plugins
 The Littlest JupyterHub can install additional *plugins* that provide additional
 features. They are most commonly used to install a particular *stack* - such as
 the `PANGEO Stack <https://github.com/yuvipanda/tljh-pangeo>`_ for earth sciences
-research, a stack for a praticular class, etc.
+research, a stack for a particular class, etc.
 
 ``--plugin <plugin-to-install>`` installs and activates a plugin. You can pass it
 however many times you want. Since plugins are distributed as python packages,
 ``<plugin-to-install>`` can be anything that can be passed to ``pip install`` -
-``plugin-name-on-pypy==<version>`` and ``git+https://github.com/user/repo@tag``
+``plugin-name-on-pypi==<version>`` and ``git+https://github.com/user/repo@tag``
 are the most popular ones. Specifying a version or tag is highly recommended.
 
 For example, to install the PANGEO Plugin version 0.1 in your new TLJH install,
@@ -84,4 +84,4 @@ you would use:
 .. note::
 
    Plugins are extremely powerful and can do a large number of arbitrary things.
-   Only install plugins you trust.
+   Only install plugins you trust.

docs/topic/idle-culler.rst

@@ -0,0 +1,114 @@
+.. _topic/idle-culler:
+
+=============================
+Culling idle notebook servers
+=============================
+
+The idle culler automatically shuts down user notebook servers when they have
+not been used for a certain time period, in order to reduce the total resource
+usage on your JupyterHub.
+
+
+JupyterHub pings the user's notebook server at certain time intervals. If no response
+is received from the server during this checks and the timeout expires, the server is
+considered to be *inactive (idle)* and will be culled.
+
+
+Default settings
+================
+
+By default, JupyterHub will ping the user notebook servers every 60s to check their
+status. Every server found to be idle for more than 10 minutes will be culled.
+
+.. code-block:: python
+
+	services.cull.every = 60
+	services.cull.timeout = 600
+
+Because the servers don't have a maximum age set, an active server will not be shut down
+regardless of how long it has been up and running.
+
+.. code-block:: python
+
+	services.cull.max_age = 0
+
+If after the culling process, there are users with no active notebook servers, by default,
+the users will not be culled alongside their notebooks and will continue to exist.
+
+.. code-block:: python
+
+	services.cull.users = False
+
+
+Configuring the idle culler
+===========================
+
+The available configuration options are:
+
+Idle timeout
+------------
+The idle timeout is the maximum time (in seconds) a server can be inactive before it
+will be culled. The timeout can be configured using:
+
+.. code-block:: bash
+
+	sudo tljh-config set services.cull.timeout <max-idle-sec-before-server-is-culled>
+	sudo tljh-config reload
+
+Idle check interval
+-------------------
+The idle check interval represents how frequent (in seconds) the Hub will
+check if there are any idle servers to cull. It can be configured using:
+
+.. code-block:: bash
+
+ 	sudo tljh-config set services.cull.every <number-of-sec-this-check-is-done>
+ 	sudo tljh-config reload
+
+Maximum age
+-----------
+The maximum age sets the time (in seconds) a server should be running.
+The servers that exceed the maximum age, will be culled even if they are active.
+A maximum age of 0, will deactivate this option.
+The maximum age can be configured using:
+
+.. code-block:: bash
+
+ 	sudo tljh-config set services.cull.max_age <server-max-age>
+ 	sudo tljh-config reload
+
+User culling
+------------
+In addition to servers, it is also possible to cull the users. This is usually
+suited for temporary-user cases such as *tmpnb*.
+User culling can be activated using the following command:
+
+.. code-block:: bash
+
+ 	sudo tljh-config set services.cull.users True
+ 	sudo tljh-config reload
+
+Concurrency
+-----------
+Deleting a lot of users at the same time can slow down the Hub.
+The number of concurrent requests made to the Hub can be configured using:
+
+.. code-block:: bash
+
+ 	sudo tljh-config set services.cull.concurrency <number-of-concurrent-hub-requests>
+ 	sudo tljh-config reload
+
+Because TLJH it's used for a small number of users, the cases that may require to
+modify the concurrency limit should be rare.
+
+
+Disabling the idle culler
+=========================
+
+The idle culling service is enabled by default. To disable it, use the following
+command:
+
+.. code-block:: bash
+
+   sudo tljh-config set services.cull.enabled False
+   sudo tljh-config reload

docs/topic/installer-actions.rst

@@ -51,8 +51,8 @@ By default, ``sudo`` does not respect any custom environments you have activated
 ``tljh-config`` symlink
 ========================
 
-We create a symlink from ``/usr/bin/tljh-config`` to ``/opt/tljh/hub/bin/tljh-cohnfig``, so users
-can run ``sudo tljh-config <somethihng>`` from their terminal. While the user environment is added
+We create a symlink from ``/usr/bin/tljh-config`` to ``/opt/tljh/hub/bin/tljh-config``, so users
+can run ``sudo tljh-config <something>`` from their terminal. While the user environment is added
 to users' ``$PATH`` when they launch through JupyterHub, the hub environment is not. This makes it
 hard to access the ``tljh-config`` command used to change most config parameters. Hence we symlink the
 ``tljh-config`` command to ``/usr/local/bin``, so it is directly accessible with ``sudo tljh-config <command>``.

docs/topic/security.rst

@@ -2,9 +2,9 @@
 Security Considerations
 =======================
 
-The Littlest JupyterHub is in pre-alpha state & should not be used in
-security critical situations. We will try to keep things as secure as possible,
-but sometimes trade security for massive gains in convenience. This page contains
+The Littlest JupyterHub is in beta state & should not be used in security
+critical situations. We will try to keep things as secure as possible, but
+sometimes trade security for massive gains in convenience. This page contains
 information about the security model of The Littlest JupyterHub.
 
 System user accounts

docs/topic/tljh-config.rst

@@ -8,19 +8,22 @@ Configuring TLJH with ``tljh-config``
 changes to TLJH.
 
 Running ``tljh-config``
-======================`
+=======================
 
 You can run ``tljh-config`` in two ways:
 
 #. From inside a terminal in JupyterHub while logged in as an admin user.
-   This method is **recommended**.
+   This method is recommended.
 
 #. By directly calling ``/opt/tljh/hub/bin/tljh-config`` as root when
    logged in to the server via other means (such as SSH). This is an
    advanced use case, and not covered much in this guide.
 
-Set a configuration property
-============================
+.. _tljh-set:
+
+
+Set / Unset a configuration property
+====================================
 
 TLJH's configuration is organized in a nested tree structure. You can
 set a particular property with the following command:
@@ -47,15 +50,51 @@ do so with the following:
 
 This can only set string and numerical properties, not lists.
 
+To unset a configuration property you can use the following command:
+
+.. code-block:: bash
+
+   sudo tljh-config unset <property-path>
+
+Unsetting a configuration property removes the property from the configuration
+file. If what you want is only to change the property's value, you should use
+``set`` and overwrite it with the desired value.
+
+
 Some of the existing ``<property-path>`` are listed below by categories:
 
-**Authentication**
+
+.. _tljh-set-auth:
+
+Authentication
+--------------
 
     Use ``auth.type`` to determine authenticator to use. All parameters
     in the config under ``auth.{auth.type}`` will be passed straight to the
     authenticators themselves.
+    
+.. _tljh-set-ports:
+
+Ports
+-----
+
+  Use ``http.port`` and ``https.port`` to set the ports that TLJH will listen on, 
+  which are 80 and 443 by default. However, if you change these, note that 
+  TLJH does a lot of other things to the system (with user accounts and sudo
+  rules primarily) that might break security assumptions your other 
+  applications have, so use with extreme caution.
+  
+  .. code-block:: bash
+
+    sudo tljh-config set http.port 8080
+    sudo tljh-config set https.port 8443
+    sudo tljh-config reload proxy
+
+.. _tljh-set-user-lists:
+
+User Lists
+----------
 
-**User Lists**
 
 * ``users.allowed`` takes in usernames to whitelist
 
@@ -63,10 +102,15 @@ Some of the existing ``<property-path>`` are listed below by categories:
 
 * ``users.admin`` takes in usernames to designate as admins
 
-**User Server Limits**
+.. _tljh-set-user-limits:
+
+User Server Limits
+------------------
+
 
 * ``limits.memory`` Specifies the maximum memory that can be used by each
-  individual user. It can be specified as an absolute byte value. You can use
+  individual user. By default there is no memory limit. The limit can be
+  specified as an absolute byte value. You can use
   the suffixes K, M, G or T to mean Kilobyte, Megabyte, Gigabyte or Terabyte
   respectively. Setting it to ``None`` disables memory limits.
 
@@ -80,6 +124,7 @@ Some of the existing ``<property-path>`` are listed below by categories:
   handedly take down the machine accidentally by OOMing it.
 
 * ``limits.cpu`` A float representing the total CPU-cores each user can use.
+  By default there is no CPU limit.
   1 represents one full CPU, 4 represents 4 full CPUs, 0.5 represents
   half of one CPU, etc. This value is ultimately converted to a percentage and
   rounded down to the nearest integer percentage,
@@ -90,7 +135,11 @@ Some of the existing ``<property-path>`` are listed below by categories:
 
      sudo tljh-config set limits.cpu 2
 
-**User Environment**
+.. _tljh-set-user-env:
+
+User Environment
+----------------
+
 
     ``user_environment.default_app`` Set default application users are
     launched into. Currently can be set to the following values
@@ -100,6 +149,36 @@ Some of the existing ``<property-path>`` are listed below by categories:
 
        sudo tljh-config set user_environment.default_app jupyterlab
 
+.. _tljh-set-extra-user-groups:
+
+Extra User Groups
+=================
+
+
+``users.extra_user_groups`` is a configuration option that can be used
+to automatically add a user to a specific group. By default, there are
+no extra groups defined.
+
+Users can be "paired" with the desired, **existing** groups using:
+
+* ``tljh-config set``, if only one user is to be added to the
+  desired group:
+
+.. code-block:: bash
+
+  tljh-config set users.extra_user_groups.group1 user1
+
+* ``tljh-config add-item``, if multiple users are to be added to
+  the group:
+
+.. code-block:: bash
+
+  tljh-config add-item users.extra_user_groups.group1 user1
+  tljh-config add-item users.extra_user_groups.group1 user2
+
+
+.. _tljh-view-conf:
+
 View current configuration
 ==========================
 
@@ -112,6 +191,9 @@ To see the current configuration, you can run the following command:
 This will print the current configuration of your TLJH. This is very
 useful when asking for support!
 
+.. _tljh-reload-hub:
+
+
 Reloading JupyterHub to apply configuration
 ===========================================
 
@@ -125,10 +207,12 @@ it to take effect. You can do so with:
 This should not affect any running users. The JupyterHub will be
 restarted and loaded with the new configuration.
 
+.. _tljh-edit-yaml:
+
 Advanced: ``config.yaml``
 =========================
 
 ``tljh-config`` is a simple program that modifies the contents of the
-``config.yaml`` file located at ``/opt/tljh/config.yaml``. ``tljh-config``
+``config.yaml`` file located at ``/opt/tljh/config/config.yaml``. ``tljh-config``
 is the recommended method of editing / viewing configuration since editing
 YAML by hand in a terminal text editor is a large source of errors.

docs/troubleshooting/logs.rst

@@ -44,18 +44,20 @@ logs is a great first step.
 This command displays logs from JupyterHub itself. See :ref:`journalctl_tips`
 for tips on navigating the logs.
 
-Configurable HTTP Proxy Logs
-============================
+.. _troubleshooting/logs/traefik:
 
-Configurable HTTP Proxy redirects traffic to JupyterHub / user notebook servers
-as necessary & handles HTTPS. It usually is the least problematic of the components,
-but things do go wrong sometimes!
+Traefik Proxy Logs
+==================
+
+`traefik <https://traefik.io/>`_ redirects traffic to JupyterHub / user notebook servers
+as necessary & handles HTTPS. Look at this if all you can see in your browser
+is one line cryptic error messages, or if you are having trouble with HTTPS.
 
 .. code-block:: bash
 
-   sudo journalctl -u configurable-http-proxy
+   sudo journalctl -u traefik
 
-This command displays logs from Configurable HTTP Proxy. See :ref:`journalctl_tips`
+This command displays logs from Traefik. See :ref:`journalctl_tips`
 for tips on navigating the logs.
 
 User Server Logs

docs/troubleshooting/providers/amazon.rst

@@ -0,0 +1,32 @@
+=============================================
+Troubleshooting issues on Amazon Web Services
+=============================================
+
+This is an incomplete list of issues people have run into when running
+TLJH on Amazon Web Services (AWS), and how they have fixed them!
+
+'Connection Refused' error after restarting server
+==================================================
+
+If you restarted your server from the EC2 Management Console & then try to access
+your JupyterHub from a browser, you might get a **Connection Refused** error.
+This is most likely because the **External IP** of your server has changed.
+
+Check the **IPv4 Public IP** dislayed in the bottom of the `EC2 Management Console` 
+screen for that instance matches the IP you are trying to access. If you have a 
+domain name pointing to the IP address, you might have to change it to point to 
+the new correct IP.
+
+You can prevent public IP changes by `associating a static IP
+<https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html>`_
+with your server. In the Amazon Web Services ecosystem, the public static IP 
+addresses are handled under `Elastic IP addresses` category of AWS; these 
+addresses are tied to the overall AWS account. `This guide 
+<https://dzone.com/articles/assign-fixed-ip-aws-ec2>`_ might be helpful. Notice 
+there can be a cost to this. Although `the guide 
+<https://dzone.com/articles/assign-fixed-ip-aws-ec2>`_ is outdated (generally 
+half that `price now <https://aws.amazon.com/ec2/pricing/on-demand/#Elastic_IP_Addresses>`_), 
+Amazon describes `here <https://aws.amazon.com/premiumsupport/knowledge-center/elastic-ip-charges/>`_ 
+how the Elastic IP address feature is free when associated with a running 
+instance, but that you'll be charged by the hour for maintaining that specific
+IP address when it isn't associated with a running instance.

docs/troubleshooting/providers/custom.rst

@@ -0,0 +1,30 @@
+.. _troubleshooting/providers/custom:
+
+=========================================
+Troubleshooting issues on your own server
+=========================================
+
+This is an incomplete list of issues people have run into
+when installing TLJH on their own servers, and ways they
+have fixed them.
+
+Outgoing HTTP proxy required
+============================
+If your server is behind a firewall that requires a HTTP proxy to reach
+the internet, run these commands before running the installer
+
+.. code-block:: bash
+
+    export http_proxy=<your_proxy-server>
+
+HTTPS certificate interception
+==============================
+
+If your server is behind a firewall that intercepts HTTPS requests
+and re-signs them, you might have to explicitly tell TLJH which
+certificates to use.
+
+.. code::
+
+    export REQUESTS_CA_BUNDLE=</directory/with/your/ssl/certificates>
+    sudo npm config set cafile=</directory/with/your/ssl/certificates>

integration-tests/conftest.py

@@ -4,12 +4,14 @@ import os
 
 from pytest import fixture
 
-from tljh.config import CONFIG_FILE, reload_component
 
 
 @fixture
 def preserve_config(request):
     """Fixture to save and restore config around tests"""
+    # Import TLJH only when needed. This lets us run tests in places
+    # where TLJH is not installed - particularly, the 'distro check' test.
+    from tljh.config import CONFIG_FILE, reload_component
     if os.path.exists(CONFIG_FILE):
         with open(CONFIG_FILE) as f:
             save_config = f.read()

integration-tests/plugins/simplest/tljh_simplest.py

@@ -17,6 +17,11 @@ def tljh_extra_user_pip_packages():
         'django',
     ]
 
+@hookimpl
+def tljh_extra_hub_pip_packages():
+    return [
+        'there',
+    ]
 
 @hookimpl
 def tljh_extra_apt_packages():
@@ -30,4 +35,14 @@ def tljh_config_post_install(config):
     # Put an arbitrary marker we can test for
     config['simplest_plugin'] = {
         'present': True
-    }
+    }
+
+@hookimpl
+def tljh_custom_jupyterhub_config(c):
+    c.JupyterHub.authenticator_class = 'tmpauthenticator.TmpAuthenticator'
+
+
+@hookimpl
+def tljh_post_install():
+    with open('test_post_install', 'w') as f:
+        f.write('123456789')

integration-tests/test_bootstrap.py

@@ -0,0 +1,56 @@
+"""
+Test running bootstrap script in different circumstances
+"""
+import subprocess
+from textwrap import dedent
+
+def run_bootstrap(container_name, image):
+    # stop container if it is already running
+    subprocess.run([
+        'docker', 'rm', '-f', container_name
+    ])
+    
+    # Start a detached Ubuntu 16.04 container
+    subprocess.check_call([
+        'docker', 'run', '--detach', '--name', container_name, image,
+        '/bin/bash', '-c', 'sleep 1000s'
+    ])
+    # Install python3 inside the ubuntu container
+    # There is no trusted Ubuntu+Python3 container we can use
+    subprocess.check_output([
+        'docker', 'exec', container_name, 'apt-get', 'update'
+    ])
+    subprocess.check_output([
+        'docker', 'exec', container_name, 'apt-get', 'install', '--yes', 'python3'
+    ])
+    # Copy only the bootstrap script  to container, so this is faster
+    subprocess.check_call([
+        'docker',
+        'cp',
+        'bootstrap/', f'{container_name}:/srv'
+    ])
+
+    # Run bootstrap script, return the output
+    return subprocess.run([
+        'docker', 'exec', '-i', container_name,
+        'python3', '/srv/bootstrap/bootstrap.py'
+    ], check=False, stdout=subprocess.PIPE, encoding='utf-8')
+
+def test_ubuntu_too_old():
+    """
+    Error with a useful message when running in older Ubuntu
+    """
+    output = run_bootstrap('old-distro-test', 'ubuntu:16.04')
+    assert output.stdout == 'The Littlest JupyterHub requires Ubuntu 18.04 or higher\n'
+    assert output.returncode == 1
+
+
+def test_inside_no_systemd_docker():
+    output = run_bootstrap('plain-docker-test', 'ubuntu:18.04')
+    assert output.stdout.strip() == dedent("""
+        Systemd is required to run TLJH
+        Running inside a docker container without systemd isn't supported
+        We recommend against running a production TLJH instance inside a docker container
+        For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html
+    """).strip()
+    assert output.returncode == 1

integration-tests/test_extensions.py

@@ -13,9 +13,9 @@ def test_serverextensions():
     ], stderr=subprocess.PIPE)
 
     extensions = [
-        'jupyterlab 0.34.1',
+        'jupyterlab 0.35.4',
         'nbgitpuller 0.6.1',
-        'nteract_on_jupyter 1.8.1',
+        'nteract_on_jupyter 2.0.7',
         'nbresuse '
     ]
 
@@ -34,13 +34,15 @@ def test_nbextensions():
 
     extensions = [
         'nbresuse/main',
+        # This is what ipywidgets nbextension is called
+        'jupyter-js-widgets/extension'
     ]
 
     for e in extensions:
         assert '{} \x1b[32m enabled \x1b[0m'.format(e) in proc.stdout.decode()
 
     # Ensure we have 'OK' messages in our stdout, to make sure everything is importable
-    proc.stderr.decode() == '      - Validating: \x1b[32mOK\x1b[0m\n' * len(extensions)
+    assert proc.stderr.decode() == '      - Validating: \x1b[32mOK\x1b[0m\n' * len(extensions)
 
 
 def test_labextensions():

integration-tests/test_hub.py

@@ -1,6 +1,7 @@
 import requests
 from hubtraf.user import User
 from hubtraf.auth.dummy import login_dummy
+from jupyterhub.utils import exponential_backoff
 import secrets
 import pytest
 from functools import partial
@@ -9,6 +10,7 @@ import pwd
 import grp
 import sys
 import subprocess
+from os import system
 from tljh.normalize import generate_system_username
 
 
@@ -34,10 +36,6 @@ async def test_user_code_execute():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
-
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -62,9 +60,6 @@ async def test_user_admin_add():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -76,7 +71,9 @@ async def test_user_admin_add():
             assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem
 
 
+# FIXME: Make this test pass
 @pytest.mark.asyncio
+@pytest.mark.xfail(reason="Unclear why this is failing")
 async def test_user_admin_remove():
     """
     User is made an admin, logs in and we check if they are in admin group.
@@ -92,9 +89,6 @@ async def test_user_admin_remove():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -105,16 +99,14 @@ async def test_user_admin_remove():
             # Assert that the user has admin rights
             assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem
 
-
             assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait()
             assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
-            await asyncio.sleep(1)
 
             await u.stop_server()
             await u.ensure_server()
 
             # Assert that the user does *not* have admin rights
-            assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem
+            assert f'jupyter-{username}' not in grp.getgrnam('jupyterhub-admins').gr_mem
 
 
 @pytest.mark.asyncio
@@ -130,9 +122,6 @@ async def test_long_username():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     try:
         async with User(username, hub_url, partial(login_dummy, password='')) as u:
                 await u.login()
@@ -150,4 +139,141 @@ async def test_long_username():
             '-u', 'jupyterhub',
             '--no-pager'
         ])
-        raise
+        raise
+
+
+@pytest.mark.asyncio
+async def test_user_group_adding():
+    """
+    User logs in, and we check if they are added to the specified group.
+    """
+    # This *must* be localhost, not an IP
+    # aiohttp throws away cookies if we are connecting to an IP!
+    hub_url = 'http://localhost'
+    username = secrets.token_hex(8)
+    groups = {"somegroup": [username]}
+    # Create the group we want to add the user to
+    system('groupadd somegroup')
+
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
+
+    try:
+        async with User(username, hub_url, partial(login_dummy, password='')) as u:
+                await u.login()
+                await u.ensure_server()
+
+                # Assert that the user exists
+                system_username = generate_system_username(f'jupyter-{username}')
+                assert pwd.getpwnam(system_username) is not None
+
+                # Assert that the user was added to the specified group
+                assert f'jupyter-{username}' in grp.getgrnam('somegroup').gr_mem
+
+                await u.stop_server()
+                # Delete the group
+                system('groupdel somegroup')
+    except:
+        # If we have any errors, print jupyterhub logs before exiting
+        subprocess.check_call([
+            'journalctl',
+            '-u', 'jupyterhub',
+            '--no-pager'
+        ])
+        raise
+
+
+@pytest.mark.asyncio
+async def test_idle_server_culled():
+    """
+    User logs in, starts a server & stays idle for 1 min.
+    (the user's server should be culled during this period)
+    """
+    # This *must* be localhost, not an IP
+    # aiohttp throws away cookies if we are connecting to an IP!
+    hub_url = 'http://localhost'
+    username = secrets.token_hex(8)
+
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    # Check every 10s for idle servers to cull
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
+    # Apart from servers, also cull users
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait()
+    # Cull servers and users after 60s of activity
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
+
+    async with User(username, hub_url, partial(login_dummy, password='')) as u:
+            await u.login()
+            # Start user's server
+            await u.ensure_server()
+            # Assert that the user exists
+            assert pwd.getpwnam(f'jupyter-{username}') is not None
+
+            # Check that we can get to the user's server
+            r = await u.session.get(u.hub_url / 'hub/api/users' / username,
+                headers={'Referer': str(u.hub_url / 'hub/')})
+            assert r.status == 200
+
+            async def _check_culling_done():
+                # Check that after 60s, the user and server have been culled and are not reacheable anymore
+                r = await u.session.get(u.hub_url / 'hub/api/users' / username,
+                    headers={'Referer': str(u.hub_url / 'hub/')})
+                print(r.status)
+                return r.status == 403
+
+            await exponential_backoff(
+                _check_culling_done,
+                "Server culling failed!",
+                timeout=100,
+            )
+
+@pytest.mark.asyncio
+async def test_active_server_not_culled():
+    """
+    User logs in, starts a server & stays idle for 30s
+    (the user's server should not be culled during this period).
+    """
+    # This *must* be localhost, not an IP
+    # aiohttp throws away cookies if we are connecting to an IP!
+    hub_url = 'http://localhost'
+    username = secrets.token_hex(8)
+
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    # Check every 10s for idle servers to cull
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
+    # Apart from servers, also cull users
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait()
+    # Cull servers and users after 60s of activity
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
+
+    async with User(username, hub_url, partial(login_dummy, password='')) as u:
+            await u.login()
+            # Start user's server
+            await u.ensure_server()
+            # Assert that the user exists
+            assert pwd.getpwnam(f'jupyter-{username}') is not None
+
+            # Check that we can get to the user's server
+            r = await u.session.get(u.hub_url / 'hub/api/users' / username,
+                headers={'Referer': str(u.hub_url / 'hub/')})
+            assert r.status == 200
+
+            async def _check_culling_done():
+                # Check that after 30s, we can still reach the user's server
+                r = await u.session.get(u.hub_url / 'hub/api/users' / username,
+                    headers={'Referer': str(u.hub_url / 'hub/')})
+                print(r.status)
+                return r.status != 200
+
+            try:
+                await exponential_backoff(
+                    _check_culling_done,
+                    "User's server is still reacheable!",
+                    timeout=30,
+                )
+            except TimeoutError:
+                # During the 30s timeout the user's server wasn't culled, which is what we intended.
+                pass

integration-tests/test_install.py

@@ -117,6 +117,12 @@ def test_admin_writable():
     permissions_test(ADMIN_GROUP, sys.prefix, writable=True, dirs_only=True)
 
 
+def test_installer_log_readable():
+    # Test that installer.log is owned by root, and not readable by anyone else
+    file_stat = os.stat('/opt/tljh/installer.log')
+    assert file_stat.st_uid == 0
+    assert file_stat.st_mode == 0o100500
+
 @pytest.mark.parametrize("group", [ADMIN_GROUP, USER_GROUP])
 def test_user_env_readable(group):
     # every file in user env should be readable by everyone

integration-tests/test_proxy.py

@@ -53,8 +53,13 @@ def test_manual_https(preserve_config):
     # verify that our certificate was loaded by traefik
     assert server_cert == file_cert
 
-    # verify that we can still connect to the hub
-    r = requests.get("https://127.0.0.1/hub/api", verify=False)
+    for i in range(5):
+        time.sleep(i)
+        # verify that we can still connect to the hub
+        r = requests.get("https://127.0.0.1/hub/api", verify=False)
+        if r.status_code == 200:
+            break;
+
     r.raise_for_status()
 
     # cleanup