From fc0ecb66993dfd77b4adf4bf48e852f0e29a0264 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Thu, 19 Jul 2018 17:30:09 -0700 Subject: [PATCH] Use venv for base hub environment - TLJH should support raspberry pi, which runs ARM. conda does not support ARM. - Get nodejs from nodesource instead of conda or default repositories. Default repositories get out of date pretty quickly. - Install CHP from npm --- bootstrap/bootstrap.py | 120 +++--------------- tests/test_installer.py | 10 ++ tljh/apt.py | 43 +++++++ tljh/conda.py | 77 ++++++++++- tljh/installer.py | 90 ++++++++++++- .../configurable-http-proxy.service | 4 +- 6 files changed, 227 insertions(+), 117 deletions(-) create mode 100644 tests/test_installer.py create mode 100644 tljh/apt.py diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 7d23db3..9654ada 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -11,131 +11,45 @@ Constraints: - Be compatible with Python 3.4 (since we support Ubuntu 16.04) - Use stdlib modules only """ -from distutils.version import LooseVersion as V import os import subprocess -import urllib.request -import contextlib -import hashlib -import tempfile import sys -def md5_file(fname): - """ - Return md5 of a given filename - - Copied from https://stackoverflow.com/a/3431838 - """ - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - - -def check_miniconda_version(prefix, version): - """ - Return true if a miniconda install with version exists at prefix - """ - try: - installed_version = subprocess.check_output([ - os.path.join(prefix, 'bin', 'conda'), - '-V' - ]).decode().strip().split()[1] - except (subprocess.CalledProcessError, FileNotFoundError): - # Conda doesn't exist, or wrong version - return False - else: - return V(installed_version) >= V(version) - - -@contextlib.contextmanager -def download_miniconda_installer(version, md5sum): - """ - Context manager to download miniconda installer of given version. - - This should be used as a contextmanager. It downloads miniconda installer - of given version, verifies the md5sum & provides path to it to the `with` - block to run. - """ - with tempfile.NamedTemporaryFile() as f: - installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) - urllib.request.urlretrieve(installer_url, f.name) - - if md5_file(f.name) != md5sum: - raise Exception('md5 hash mismatch! Downloaded file corrupted') - - yield f.name - - -def install_miniconda(installer_path, prefix): - """ - Install miniconda with installer at installer_path under prefix - """ - subprocess.check_output([ - '/bin/bash', - installer_path, - '-u', '-b', - '-p', prefix - ], stderr=subprocess.STDOUT) - # fix permissions on initial install - # a few files have the wrong ownership and permissions initially - # when the installer is run as root - subprocess.check_call( - ["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix] - ) - subprocess.check_call(["chmod", "-R", "o-w", prefix]) - - -def pip_install(prefix, packages, editable=False): - """ - Install pip packages in the conda environment under prefix. - - Packages are upgraded if possible. - - Set editable=True to add '--editable' to the pip install commandline. - Very useful when doing active development - """ - flags = ['--upgrade'] - if editable: - flags.append('--editable') - subprocess.check_output([ - os.path.join(prefix, 'bin', 'python3'), - '-m', 'pip', - 'install', - ] + flags + packages) - - def main(): install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') - miniconda_version = '4.5.4' - miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" print('Checking if TLJH is already installed...') - if not check_miniconda_version(hub_prefix, miniconda_version): - initial_setup = True - print('Downloading & setting up hub environment...') - with download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: - install_miniconda(installer_path, hub_prefix) - print('Hub environment set up!') - else: + if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): + print('TLJH already installed, upgrading...') initial_setup = False - print('TLJH is already installed, will try to upgrade') + else: + print('Setting up hub environment') + initial_setup = True + subprocess.check_output(['apt-get', 'update', '--yes']) + # gnupg2 is needed for nodesource to work + subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'gnupg2']) + os.makedirs(hub_prefix, exist_ok=True) + subprocess.check_output(['python3', '-m', 'venv', hub_prefix]) if initial_setup: print('Setting up TLJH installer...') else: print('Upgrading TLJH installer...') - is_dev = os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes' + pip_flags = ['--upgrade'] + if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes': + pip_flags.append('--editable') tljh_repo_path = os.environ.get( 'TLJH_BOOTSTRAP_PIP_SPEC', 'git+https://github.com/yuvipanda/the-littlest-jupyterhub.git' ) - pip_install(hub_prefix, [tljh_repo_path], editable=is_dev) + subprocess.check_output([ + os.path.join(hub_prefix, 'bin', 'pip'), + 'install' + ] + pip_flags + [tljh_repo_path]) print('Starting TLJH installer...') os.execv( diff --git a/tests/test_installer.py b/tests/test_installer.py new file mode 100644 index 0000000..b8a7834 --- /dev/null +++ b/tests/test_installer.py @@ -0,0 +1,10 @@ +""" +Unit test functions in installer.py +""" +from tljh import installer +import os + + +def test_ensure_node(): + installer.ensure_node() + assert os.path.exists('/usr/bin/node') diff --git a/tljh/apt.py b/tljh/apt.py new file mode 100644 index 0000000..d6ac08e --- /dev/null +++ b/tljh/apt.py @@ -0,0 +1,43 @@ +""" +Utilities for working with the apt package manager +""" +import os +import subprocess + + +def trust_gpg_key(key): + """ + Trust given GPG public key. + + key is a GPG public key (bytes) that can be passed to apt-key add via stdin. + """ + subprocess.run(['apt-key', 'add', '-'], input=key, check=True) + + +def add_source(name, source_url, section): + """ + Add a debian package source. + + distro is determined from /etc/os-release + """ + # lsb_release is not installed in most docker images by default + distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}']).decode().strip() + line = f'deb {source_url} {distro} {section}' + with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f: + # Write out deb line only if it already doesn't exist + if f.read() != line: + f.seek(0) + f.write(line) + f.truncate() + subprocess.check_output(['apt-get', 'update', '--yes']) + + +def install_packages(packages): + """ + Install debian packages + """ + subprocess.check_output([ + 'apt-get', + 'install', + '--yes' + ] + packages) diff --git a/tljh/conda.py b/tljh/conda.py index d7dab54..02e28cc 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -4,20 +4,86 @@ Wrap conda commandline program import os import subprocess import json -import sys +import hashlib +import contextlib +import tempfile +import urllib.request -# Use sys.executable to call conda to avoid needing to fudge PATH -CONDA_EXECUTABLE = [sys.executable, '-m', 'conda'] + +def md5_file(fname): + """ + Return md5 of a given filename + + Copied from https://stackoverflow.com/a/3431838 + """ + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def check_miniconda_version(prefix, version): + """ + Return true if a miniconda install with version exists at prefix + """ + try: + return subprocess.check_output([ + os.path.join(prefix, 'bin', 'conda'), + '-V' + ]).decode().strip() == 'conda {}'.format(version) + except (subprocess.CalledProcessError, FileNotFoundError): + # Conda doesn't exist, or wrong version + return False + + +@contextlib.contextmanager +def download_miniconda_installer(version, md5sum): + """ + Context manager to download miniconda installer of given version. + + This should be used as a contextmanager. It downloads miniconda installer + of given version, verifies the md5sum & provides path to it to the `with` + block to run. + """ + with tempfile.NamedTemporaryFile() as f: + installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) + urllib.request.urlretrieve(installer_url, f.name) + + if md5_file(f.name) != md5sum: + raise Exception('md5 hash mismatch! Downloaded file corrupted') + + yield f.name + + +def install_miniconda(installer_path, prefix): + """ + Install miniconda with installer at installer_path under prefix + """ + subprocess.check_output([ + '/bin/bash', + installer_path, + '-u', '-b', + '-p', prefix + ], stderr=subprocess.STDOUT) + # fix permissions on initial install + # a few files have the wrong ownership and permissions initially + # when the installer is run as root + subprocess.check_call( + ["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix] + ) + subprocess.check_call(["chmod", "-R", "o-w", prefix]) def ensure_conda_env(prefix): """ Ensure a conda environment in the prefix """ + conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] abspath = os.path.abspath(prefix) try: output = json.loads( - subprocess.check_output(CONDA_EXECUTABLE + ['create', '--json', '--prefix', abspath]).decode() + subprocess.check_output(conda_executable + ['create', '--json', '--prefix', abspath]).decode() ) except subprocess.CalledProcessError as e: output = json.loads(e.output.decode()) @@ -32,10 +98,11 @@ def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. """ + conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] abspath = os.path.abspath(prefix) # Let subprocess errors propagate # FIXME: raise different exception when using - raw_output = subprocess.check_output(CONDA_EXECUTABLE + [ + raw_output = subprocess.check_output(conda_executable + [ 'install', '-c', 'conda-forge', # Make customizable if we ever need to '--json', diff --git a/tljh/installer.py b/tljh/installer.py index 0a0c3cb..b7b19d9 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -9,7 +9,7 @@ from urllib.request import urlopen, URLError from ruamel.yaml import YAML -from tljh import conda, systemd, user +from tljh import conda, systemd, user, apt INSTALL_PREFIX = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'hub') @@ -21,6 +21,79 @@ HERE = os.path.abspath(os.path.dirname(__file__)) rt_yaml = YAML() +def ensure_node(): + """ + Ensure nodejs from nodesource is installed + """ + key = b""" +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 +Comment: GPGTools - https://gpgtools.org + +mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+ +W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs +fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy +qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7 +89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD +Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7 +C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/ +G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc +Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft +qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm +EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB +tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2 +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy +jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ +kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1 +GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm +XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU +VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka +1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI +IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc +pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn +xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y +gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI +AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ +fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ +Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh +41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea +JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD +xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi +vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/ +aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o +QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK +yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3 +QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek +fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK +CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec +0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1 +PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh +qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15 +ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac +hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb +DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4 +xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu +G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd +sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy +/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g== +=CLGF +-----END PGP PUBLIC KEY BLOCK----- + """.strip() + apt.trust_gpg_key(key) + apt.add_source('nodesource', f'https://deb.nodesource.com/node_10.x', 'main') + apt.install_packages(['nodejs']) + + +def ensure_chp_package(prefix): + """ + Ensure CHP is installed + """ + if not os.path.exists(os.path.join(prefix, 'node_modules', '.bin', 'configurable-http-proxy')): + subprocess.check_call([ + 'npm', 'install', 'configurable-http-proxy@3.1.0' + ], cwd=prefix) + + def ensure_jupyterhub_service(prefix): """ Ensure JupyterHub & CHP Services are set up properly @@ -70,7 +143,6 @@ def ensure_jupyterhub_package(prefix): hub environment be installed with pip prevents accidental mixing of python and conda packages! """ - conda.ensure_conda_packages(prefix, ['configurable-http-proxy==3.1.0']) conda.ensure_pip_packages(prefix, [ 'jupyterhub==0.9.1', 'jupyterhub-dummyauthenticator==0.3.1', @@ -103,7 +175,14 @@ def ensure_user_environment(user_requirements_txt_file): Set up user conda environment with required packages """ print("Setting up user environment...") - conda.ensure_conda_env(USER_ENV_PREFIX) + miniconda_version = '4.5.4' + miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" + + if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): + print('Downloading & setting up user environment...') + with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: + conda.install_miniconda(installer_path, USER_ENV_PREFIX) + conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' @@ -194,12 +273,11 @@ def main(): ensure_usergroups() ensure_user_environment(args.user_requirements_txt_url) - # Weird setuptools issue creates a few world-writable metadata files. - # Fix it: - subprocess.check_call(["chmod", "-R", "o-w", os.path.join(HUB_ENV_PREFIX, "pkgs")]) print("Setting up JupyterHub...") + ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) + ensure_chp_package(HUB_ENV_PREFIX) ensure_jupyterhub_service(HUB_ENV_PREFIX) ensure_jupyterhub_running() diff --git a/tljh/systemd-units/configurable-http-proxy.service b/tljh/systemd-units/configurable-http-proxy.service index 2b983e6..9301ac8 100644 --- a/tljh/systemd-units/configurable-http-proxy.service +++ b/tljh/systemd-units/configurable-http-proxy.service @@ -15,9 +15,7 @@ PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes EnvironmentFile={install_prefix}/state/configurable-http-proxy.secret -# Set PATH so env can find correct node -Environment=PATH=$PATH:{install_prefix}/hub/bin -ExecStart={install_prefix}/hub/bin/configurable-http-proxy \ +ExecStart={install_prefix}/hub/node_modules/.bin/configurable-http-proxy \ --ip 0.0.0.0 \ --port 80 \ --api-ip 127.0.0.1 \