From 715860707b71714415d56ff9fdac59db53979a1c Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Mon, 2 Jul 2018 15:12:26 -0700 Subject: [PATCH] Rewrite bootstrapper in Python - This was going to get too complex for bash. Only way to kill those scripts is before they get too complex. - Better progress messages from bootstrapper. - Differentiate between bootstrapper & installer - Cleanup documentation a little bit --- .circleci/config.yml | 2 +- Dockerfile | 4 +- README.rst | 13 +--- bootstrap/bootstrap.py | 125 ++++++++++++++++++++++++++++++++ docs/contributing/dev-setup.rst | 10 +-- docs/guides/install.rst | 8 +- docs/index.rst | 4 +- docs/tutorials/quickstart.rst | 2 +- installer/install.bash | 49 ------------- tljh/installer.py | 62 +++++++++------- 10 files changed, 178 insertions(+), 101 deletions(-) create mode 100644 bootstrap/bootstrap.py delete mode 100755 installer/install.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index eb9730c..591c4a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,7 +74,7 @@ jobs: name: run tljh installer command: | docker cp . tljh-systemd-ci:/srv/src - docker exec -it tljh-systemd-ci bash /srv/src/installer/install.bash + docker exec -it tljh-systemd-ci python3 /srv/src/bootstrap/bootstrap.py - run: name: check jupyterhub is up diff --git a/Dockerfile b/Dockerfile index bfebfbf..9a8cdd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,8 +21,8 @@ RUN systemctl set-default multi-user.target STOPSIGNAL SIGRTMIN+3 # Set up image to be useful out of the box for development & CI -ENV TLJH_INSTALL_PIP_FLAGS="--no-cache-dir -e" -ENV TLJH_INSTALL_PIP_SPEC=/srv/src +ENV TLJH_BOOTSTRAP_DEV=yes +ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src ENV PATH=/opt/tljh/hub/bin:${PATH} CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"] diff --git a/README.rst b/README.rst index f6fbde6..d3cbdca 100644 --- a/README.rst +++ b/README.rst @@ -18,14 +18,5 @@ more information. Quick Start ----------- -On a fresh Ubuntu 18.04 server, you can install The Littlest JupyterHub with: - -.. code-block:: bash - - curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash - - -This takes 2-5 minutes to run. When completed, you can access your new JupyterHub -at the public IP of your server (on the default http port 80)! - -For more information (including other installation methods), check out the -`documentation `_. +Install The Littlest JupyterHub (TLJH) in under 10 minutes by following the +`quickstart tutorial `_. diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py new file mode 100644 index 0000000..7da75c2 --- /dev/null +++ b/bootstrap/bootstrap.py @@ -0,0 +1,125 @@ +""" +Bootstrap an installation of TLJH. + +Sets up just enough TLJH environments to invoke tljh.installer. + +This script is run as: + + curl | sudo python3 - + +Constraints: + - Be compatible with Python 3.4 (since we support Ubuntu 16.04) + - Use stdlib modules only +""" +import os +import subprocess +import urllib.request +import contextlib +import hashlib +import tempfile + + +def md5_file(fname): + """ + Return md5 of a given filename + + Copied from https://stackoverflow.com/a/3431838 + """ + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def check_miniconda_version(prefix, version): + """ + Return true if a miniconda install with version exists at prefix + """ + try: + return subprocess.check_output([ + os.path.join(prefix, 'bin', 'conda'), + '-V' + ]).decode().strip() == 'conda {}'.format(version) + except (subprocess.CalledProcessError, FileNotFoundError): + # Conda doesn't exist, or wrong version + return False + + +@contextlib.contextmanager +def download_miniconda_installer(version): + md5sums = { + '4.5.4': "a946ea1d0c4a642ddf0c3a26a18bb16d" + } + + if version not in md5sums: + raise ValueError( + 'minicondaversion {} not supported. Supported version:'.format( + version, ' '.join(md5sums.keys()) + )) + + with tempfile.NamedTemporaryFile() as f: + installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) + urllib.request.urlretrieve(installer_url, f.name) + + if md5_file(f.name) != md5sums[version]: + raise Exception('md5 hash mismatch! Downloaded file corrupted') + + yield f.name + + +def install_miniconda(installer_path, prefix): + subprocess.check_output([ + '/bin/bash', + installer_path, + '-u', '-b', + '-p', prefix + ], stderr=subprocess.STDOUT) + + +def pip_install(prefix, packages, editable=False): + flags = '--no-cache-dir --upgrade' + if editable: + flags += '--editable' + subprocess.check_output([ + os.path.join(prefix, 'bin', 'python3'), + '-m', 'pip', + 'install', '--no-cache-dir', + ] + packages) + + +def main(): + install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') + hub_prefix = os.path.join(install_prefix, 'hub') + miniconda_version = '4.5.4' + + print('Checking if TLJH is already installed...') + if not check_miniconda_version(hub_prefix, miniconda_version): + initial_setup = True + print('Downloading & setting up hub environment...') + with download_miniconda_installer(miniconda_version) as installer_path: + install_miniconda(installer_path, hub_prefix) + print('Hub environment set up!') + else: + initial_setup = False + print('TLJH is already installed, will try to upgrade') + + if initial_setup: + print('Setting up TLJH installer...') + else: + print('Upgrading TLJH installer...') + + pip_install(hub_prefix, [ + os.environ.get('TLJH_BOOTSTRAP_PIP_SPEC', 'git+https://github.com/yuvipanda/the-littlest-jupyterhub.git') + ], editable=os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes') + + print('Starting TLJH installer...') + os.execl( + os.path.join(hub_prefix, 'bin', 'python3'), + os.path.join(hub_prefix, 'bin', 'python3'), + '-m', + 'tljh.installer' + ) + +if __name__ == '__main__': + main() diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst index ae7d2ee..42c64c0 100644 --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -34,13 +34,13 @@ The easiest & safest way to develop & test TLJH is with `Docker | sudo bash -``) makes you nervous, check out the :ref:`other installation methods ` we support! diff --git a/docs/tutorials/quickstart.rst b/docs/tutorials/quickstart.rst index c007fd5..d933735 100644 --- a/docs/tutorials/quickstart.rst +++ b/docs/tutorials/quickstart.rst @@ -25,7 +25,7 @@ Step 1: Install the Littlest JupyterHub (TLJH) .. code-block:: bash - curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash - + curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo python3 - This takes about 1-3 minutes to finish. When completed, you can visit the public IP of your server to use your JupyterHub! You can log in with any username diff --git a/installer/install.bash b/installer/install.bash deleted file mode 100755 index f7a94a4..0000000 --- a/installer/install.bash +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/bash -set -exuo pipefail - -# Set up defaults for configurable env vars -TLJH_INSTALL_PREFIX=${TLJH_INSTALL_PREFIX:-/opt/tljh} -TLJH_INSTALL_PIP_SPEC=${TLJH_INSTALL_PIP_SPEC:-git+https://github.com/yuvipanda/the-littlest-jupyterhub.git} -TLJH_INSTALL_PIP_FLAGS=${TLJH_INSTALL_PIP_FLAGS:---no-cache-dir} - - -function install_miniconda { - CONDA_DIR=${1} - CONDA_VERSION=4.5.4 - if [ -e ${CONDA_DIR}/bin/conda ]; then - if [ "$(${CONDA_DIR}/bin/conda -V)" == "conda ${CONDA_VERSION}" ]; then - # The given ${CONDA_DIR} already has a conda with given version - return - fi - fi - - URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" - INSTALLER_PATH=/tmp/miniconda-installer.sh - - curl -o ${INSTALLER_PATH} ${URL} - chmod +x ${INSTALLER_PATH} - - # Only MD5 checksums are available for miniconda - # Can be obtained from https://repo.continuum.io/miniconda/ - MD5SUM="a946ea1d0c4a642ddf0c3a26a18bb16d" - - if ! echo "${MD5SUM} ${INSTALLER_PATH}" | md5sum --quiet -c -; then - echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!" - exit 1 - fi - - bash ${INSTALLER_PATH} -u -b -p ${CONDA_DIR} - - # Allow easy direct installs from conda forge - ${CONDA_DIR}/bin/conda config --system --add channels conda-forge - - # Do not attempt to auto update conda or dependencies - ${CONDA_DIR}/bin/conda config --system --set auto_update_conda false - ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true -} - -HUB_CONDA_DIR=${TLJH_INSTALL_PREFIX}/hub -install_miniconda ${HUB_CONDA_DIR} -${HUB_CONDA_DIR}/bin/pip install --upgrade ${TLJH_INSTALL_PIP_FLAGS} ${TLJH_INSTALL_PIP_SPEC} - -${HUB_CONDA_DIR}/bin/python3 -m tljh.installer diff --git a/tljh/installer.py b/tljh/installer.py index bc88936..e957b60 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -14,8 +14,7 @@ HERE = os.path.abspath(os.path.dirname(__file__)) def ensure_jupyterhub_service(prefix): """ - Ensure JupyterHub & CHP Services are set up properly - """ + Ensure JupyterHub & CHP Services are set up properly """ with open(os.path.join(HERE, 'systemd-units', 'jupyterhub.service')) as f: hub_unit_template = f.read() @@ -66,31 +65,42 @@ def ensure_jupyterhub_package(prefix): ]) -ensure_jupyterhub_package(HUB_ENV_PREFIX) -ensure_jupyterhub_service(HUB_ENV_PREFIX) +def main(): + print("Setting up JupyterHub...") + ensure_jupyterhub_package(HUB_ENV_PREFIX) + ensure_jupyterhub_service(HUB_ENV_PREFIX) -user.ensure_group('jupyterhub-admins') -user.ensure_group('jupyterhub-users') + print("Setting up system user groups...") + user.ensure_group('jupyterhub-admins') + user.ensure_group('jupyterhub-users') -with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f: - # JupyterHub admins should have full passwordless sudo access - f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n') - # `sudo -E` should preserve the $PATH we set. This allows - # admins in jupyter terminals to do `sudo -E pip install `, - # `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env. - f.write('Defaults exempt_group = jupyterhub-admins\n') + print("Grainting passwordless sudo to JupyterHub admins...") + with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f: + # JupyterHub admins should have full passwordless sudo access + f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n') + # `sudo -E` should preserve the $PATH we set. This allows + # admins in jupyter terminals to do `sudo -E pip install `, + # `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env. + f.write('Defaults exempt_group = jupyterhub-admins\n') -conda.ensure_conda_env(USER_ENV_PREFIX) -conda.ensure_conda_packages(USER_ENV_PREFIX, [ - # Conda's latest version is on conda much more so than on PyPI. - 'conda==4.5.4' -]) + print("Setting up user environment...") + conda.ensure_conda_env(USER_ENV_PREFIX) + conda.ensure_conda_packages(USER_ENV_PREFIX, [ + # Conda's latest version is on conda much more so than on PyPI. + 'conda==4.5.4' + ]) -conda.ensure_pip_packages(USER_ENV_PREFIX, [ - # JupyterHub + notebook package are base requirements for user environment - 'jupyterhub==0.9.0', - 'notebook==5.5.0', - # Install additional notebook frontends! - 'jupyterlab==0.32.1', - 'nteract-on-jupyter==1.8.1' -]) + conda.ensure_pip_packages(USER_ENV_PREFIX, [ + # JupyterHub + notebook package are base requirements for user environment + 'jupyterhub==0.9.0', + 'notebook==5.5.0', + # Install additional notebook frontends! + 'jupyterlab==0.32.1', + 'nteract-on-jupyter==1.8.1' + ]) + + print("Done!") + + +if __name__ == '__main__': + main()