From fc0ecb66993dfd77b4adf4bf48e852f0e29a0264 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Thu, 19 Jul 2018 17:30:09 -0700 Subject: [PATCH 1/9] Use venv for base hub environment - TLJH should support raspberry pi, which runs ARM. conda does not support ARM. - Get nodejs from nodesource instead of conda or default repositories. Default repositories get out of date pretty quickly. - Install CHP from npm --- bootstrap/bootstrap.py | 120 +++--------------- tests/test_installer.py | 10 ++ tljh/apt.py | 43 +++++++ tljh/conda.py | 77 ++++++++++- tljh/installer.py | 90 ++++++++++++- .../configurable-http-proxy.service | 4 +- 6 files changed, 227 insertions(+), 117 deletions(-) create mode 100644 tests/test_installer.py create mode 100644 tljh/apt.py diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 7d23db3..9654ada 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -11,131 +11,45 @@ Constraints: - Be compatible with Python 3.4 (since we support Ubuntu 16.04) - Use stdlib modules only """ -from distutils.version import LooseVersion as V import os import subprocess -import urllib.request -import contextlib -import hashlib -import tempfile import sys -def md5_file(fname): - """ - Return md5 of a given filename - - Copied from https://stackoverflow.com/a/3431838 - """ - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - - -def check_miniconda_version(prefix, version): - """ - Return true if a miniconda install with version exists at prefix - """ - try: - installed_version = subprocess.check_output([ - os.path.join(prefix, 'bin', 'conda'), - '-V' - ]).decode().strip().split()[1] - except (subprocess.CalledProcessError, FileNotFoundError): - # Conda doesn't exist, or wrong version - return False - else: - return V(installed_version) >= V(version) - - -@contextlib.contextmanager -def download_miniconda_installer(version, md5sum): - """ - Context manager to download miniconda installer of given version. - - This should be used as a contextmanager. It downloads miniconda installer - of given version, verifies the md5sum & provides path to it to the `with` - block to run. - """ - with tempfile.NamedTemporaryFile() as f: - installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) - urllib.request.urlretrieve(installer_url, f.name) - - if md5_file(f.name) != md5sum: - raise Exception('md5 hash mismatch! Downloaded file corrupted') - - yield f.name - - -def install_miniconda(installer_path, prefix): - """ - Install miniconda with installer at installer_path under prefix - """ - subprocess.check_output([ - '/bin/bash', - installer_path, - '-u', '-b', - '-p', prefix - ], stderr=subprocess.STDOUT) - # fix permissions on initial install - # a few files have the wrong ownership and permissions initially - # when the installer is run as root - subprocess.check_call( - ["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix] - ) - subprocess.check_call(["chmod", "-R", "o-w", prefix]) - - -def pip_install(prefix, packages, editable=False): - """ - Install pip packages in the conda environment under prefix. - - Packages are upgraded if possible. - - Set editable=True to add '--editable' to the pip install commandline. - Very useful when doing active development - """ - flags = ['--upgrade'] - if editable: - flags.append('--editable') - subprocess.check_output([ - os.path.join(prefix, 'bin', 'python3'), - '-m', 'pip', - 'install', - ] + flags + packages) - - def main(): install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') - miniconda_version = '4.5.4' - miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" print('Checking if TLJH is already installed...') - if not check_miniconda_version(hub_prefix, miniconda_version): - initial_setup = True - print('Downloading & setting up hub environment...') - with download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: - install_miniconda(installer_path, hub_prefix) - print('Hub environment set up!') - else: + if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): + print('TLJH already installed, upgrading...') initial_setup = False - print('TLJH is already installed, will try to upgrade') + else: + print('Setting up hub environment') + initial_setup = True + subprocess.check_output(['apt-get', 'update', '--yes']) + # gnupg2 is needed for nodesource to work + subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'gnupg2']) + os.makedirs(hub_prefix, exist_ok=True) + subprocess.check_output(['python3', '-m', 'venv', hub_prefix]) if initial_setup: print('Setting up TLJH installer...') else: print('Upgrading TLJH installer...') - is_dev = os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes' + pip_flags = ['--upgrade'] + if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes': + pip_flags.append('--editable') tljh_repo_path = os.environ.get( 'TLJH_BOOTSTRAP_PIP_SPEC', 'git+https://github.com/yuvipanda/the-littlest-jupyterhub.git' ) - pip_install(hub_prefix, [tljh_repo_path], editable=is_dev) + subprocess.check_output([ + os.path.join(hub_prefix, 'bin', 'pip'), + 'install' + ] + pip_flags + [tljh_repo_path]) print('Starting TLJH installer...') os.execv( diff --git a/tests/test_installer.py b/tests/test_installer.py new file mode 100644 index 0000000..b8a7834 --- /dev/null +++ b/tests/test_installer.py @@ -0,0 +1,10 @@ +""" +Unit test functions in installer.py +""" +from tljh import installer +import os + + +def test_ensure_node(): + installer.ensure_node() + assert os.path.exists('/usr/bin/node') diff --git a/tljh/apt.py b/tljh/apt.py new file mode 100644 index 0000000..d6ac08e --- /dev/null +++ b/tljh/apt.py @@ -0,0 +1,43 @@ +""" +Utilities for working with the apt package manager +""" +import os +import subprocess + + +def trust_gpg_key(key): + """ + Trust given GPG public key. + + key is a GPG public key (bytes) that can be passed to apt-key add via stdin. + """ + subprocess.run(['apt-key', 'add', '-'], input=key, check=True) + + +def add_source(name, source_url, section): + """ + Add a debian package source. + + distro is determined from /etc/os-release + """ + # lsb_release is not installed in most docker images by default + distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}']).decode().strip() + line = f'deb {source_url} {distro} {section}' + with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f: + # Write out deb line only if it already doesn't exist + if f.read() != line: + f.seek(0) + f.write(line) + f.truncate() + subprocess.check_output(['apt-get', 'update', '--yes']) + + +def install_packages(packages): + """ + Install debian packages + """ + subprocess.check_output([ + 'apt-get', + 'install', + '--yes' + ] + packages) diff --git a/tljh/conda.py b/tljh/conda.py index d7dab54..02e28cc 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -4,20 +4,86 @@ Wrap conda commandline program import os import subprocess import json -import sys +import hashlib +import contextlib +import tempfile +import urllib.request -# Use sys.executable to call conda to avoid needing to fudge PATH -CONDA_EXECUTABLE = [sys.executable, '-m', 'conda'] + +def md5_file(fname): + """ + Return md5 of a given filename + + Copied from https://stackoverflow.com/a/3431838 + """ + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def check_miniconda_version(prefix, version): + """ + Return true if a miniconda install with version exists at prefix + """ + try: + return subprocess.check_output([ + os.path.join(prefix, 'bin', 'conda'), + '-V' + ]).decode().strip() == 'conda {}'.format(version) + except (subprocess.CalledProcessError, FileNotFoundError): + # Conda doesn't exist, or wrong version + return False + + +@contextlib.contextmanager +def download_miniconda_installer(version, md5sum): + """ + Context manager to download miniconda installer of given version. + + This should be used as a contextmanager. It downloads miniconda installer + of given version, verifies the md5sum & provides path to it to the `with` + block to run. + """ + with tempfile.NamedTemporaryFile() as f: + installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version) + urllib.request.urlretrieve(installer_url, f.name) + + if md5_file(f.name) != md5sum: + raise Exception('md5 hash mismatch! Downloaded file corrupted') + + yield f.name + + +def install_miniconda(installer_path, prefix): + """ + Install miniconda with installer at installer_path under prefix + """ + subprocess.check_output([ + '/bin/bash', + installer_path, + '-u', '-b', + '-p', prefix + ], stderr=subprocess.STDOUT) + # fix permissions on initial install + # a few files have the wrong ownership and permissions initially + # when the installer is run as root + subprocess.check_call( + ["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix] + ) + subprocess.check_call(["chmod", "-R", "o-w", prefix]) def ensure_conda_env(prefix): """ Ensure a conda environment in the prefix """ + conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] abspath = os.path.abspath(prefix) try: output = json.loads( - subprocess.check_output(CONDA_EXECUTABLE + ['create', '--json', '--prefix', abspath]).decode() + subprocess.check_output(conda_executable + ['create', '--json', '--prefix', abspath]).decode() ) except subprocess.CalledProcessError as e: output = json.loads(e.output.decode()) @@ -32,10 +98,11 @@ def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. """ + conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] abspath = os.path.abspath(prefix) # Let subprocess errors propagate # FIXME: raise different exception when using - raw_output = subprocess.check_output(CONDA_EXECUTABLE + [ + raw_output = subprocess.check_output(conda_executable + [ 'install', '-c', 'conda-forge', # Make customizable if we ever need to '--json', diff --git a/tljh/installer.py b/tljh/installer.py index 0a0c3cb..b7b19d9 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -9,7 +9,7 @@ from urllib.request import urlopen, URLError from ruamel.yaml import YAML -from tljh import conda, systemd, user +from tljh import conda, systemd, user, apt INSTALL_PREFIX = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'hub') @@ -21,6 +21,79 @@ HERE = os.path.abspath(os.path.dirname(__file__)) rt_yaml = YAML() +def ensure_node(): + """ + Ensure nodejs from nodesource is installed + """ + key = b""" +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 +Comment: GPGTools - https://gpgtools.org + +mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+ +W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs +fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy +qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7 +89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD +Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7 +C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/ +G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc +Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft +qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm +EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB +tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2 +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy +jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ +kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1 +GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm +XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU +VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka +1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI +IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc +pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn +xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y +gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI +AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ +fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ +Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh +41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea +JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD +xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi +vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/ +aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o +QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK +yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3 +QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek +fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK +CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec +0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1 +PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh +qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15 +ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac +hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb +DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4 +xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu +G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd +sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy +/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g== +=CLGF +-----END PGP PUBLIC KEY BLOCK----- + """.strip() + apt.trust_gpg_key(key) + apt.add_source('nodesource', f'https://deb.nodesource.com/node_10.x', 'main') + apt.install_packages(['nodejs']) + + +def ensure_chp_package(prefix): + """ + Ensure CHP is installed + """ + if not os.path.exists(os.path.join(prefix, 'node_modules', '.bin', 'configurable-http-proxy')): + subprocess.check_call([ + 'npm', 'install', 'configurable-http-proxy@3.1.0' + ], cwd=prefix) + + def ensure_jupyterhub_service(prefix): """ Ensure JupyterHub & CHP Services are set up properly @@ -70,7 +143,6 @@ def ensure_jupyterhub_package(prefix): hub environment be installed with pip prevents accidental mixing of python and conda packages! """ - conda.ensure_conda_packages(prefix, ['configurable-http-proxy==3.1.0']) conda.ensure_pip_packages(prefix, [ 'jupyterhub==0.9.1', 'jupyterhub-dummyauthenticator==0.3.1', @@ -103,7 +175,14 @@ def ensure_user_environment(user_requirements_txt_file): Set up user conda environment with required packages """ print("Setting up user environment...") - conda.ensure_conda_env(USER_ENV_PREFIX) + miniconda_version = '4.5.4' + miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" + + if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): + print('Downloading & setting up user environment...') + with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: + conda.install_miniconda(installer_path, USER_ENV_PREFIX) + conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' @@ -194,12 +273,11 @@ def main(): ensure_usergroups() ensure_user_environment(args.user_requirements_txt_url) - # Weird setuptools issue creates a few world-writable metadata files. - # Fix it: - subprocess.check_call(["chmod", "-R", "o-w", os.path.join(HUB_ENV_PREFIX, "pkgs")]) print("Setting up JupyterHub...") + ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) + ensure_chp_package(HUB_ENV_PREFIX) ensure_jupyterhub_service(HUB_ENV_PREFIX) ensure_jupyterhub_running() diff --git a/tljh/systemd-units/configurable-http-proxy.service b/tljh/systemd-units/configurable-http-proxy.service index 2b983e6..9301ac8 100644 --- a/tljh/systemd-units/configurable-http-proxy.service +++ b/tljh/systemd-units/configurable-http-proxy.service @@ -15,9 +15,7 @@ PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes EnvironmentFile={install_prefix}/state/configurable-http-proxy.secret -# Set PATH so env can find correct node -Environment=PATH=$PATH:{install_prefix}/hub/bin -ExecStart={install_prefix}/hub/bin/configurable-http-proxy \ +ExecStart={install_prefix}/hub/node_modules/.bin/configurable-http-proxy \ --ip 0.0.0.0 \ --port 80 \ --api-ip 127.0.0.1 \ From e609a840c732f093bdb0ab1ce9948c0cfd7f9ea9 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Thu, 19 Jul 2018 18:38:58 -0700 Subject: [PATCH 2/9] Install gpg2 if required when adding apt keys Not installed by default in many docker containers --- bootstrap/bootstrap.py | 3 +-- tljh/apt.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 9654ada..02a9bfc 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -28,8 +28,7 @@ def main(): print('Setting up hub environment') initial_setup = True subprocess.check_output(['apt-get', 'update', '--yes']) - # gnupg2 is needed for nodesource to work - subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'gnupg2']) + subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv']) os.makedirs(hub_prefix, exist_ok=True) subprocess.check_output(['python3', '-m', 'venv', hub_prefix]) diff --git a/tljh/apt.py b/tljh/apt.py index d6ac08e..b155124 100644 --- a/tljh/apt.py +++ b/tljh/apt.py @@ -11,6 +11,9 @@ def trust_gpg_key(key): key is a GPG public key (bytes) that can be passed to apt-key add via stdin. """ + # If gpg2 doesn't exist, install it. + if not os.path.exists('/usr/bin/gpg2'): + install_packages(['gnupg2']) subprocess.run(['apt-key', 'add', '-'], input=key, check=True) From 6a1f73c2b51e83381350806b72f1937b6a5ceaaf Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Thu, 19 Jul 2018 18:51:55 -0700 Subject: [PATCH 3/9] Run apt-get update before installing if required --- tljh/apt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tljh/apt.py b/tljh/apt.py index b155124..49be946 100644 --- a/tljh/apt.py +++ b/tljh/apt.py @@ -39,6 +39,9 @@ def install_packages(packages): """ Install debian packages """ + # Check if an apt-get update is required + if len(os.listdir('/var/lib/apt/lists')) == 0: + subprocess.check_output(['apt-get', 'update', '--yes']) subprocess.check_output([ 'apt-get', 'install', From 8ff0befe00ca76df895c2c60feba109c61b0d220 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:12:03 -0700 Subject: [PATCH 4/9] Re-instate conda version check fix Lost https://github.com/jupyterhub/the-littlest-jupyterhub/pull/58 to a rebase. Put it back! --- tljh/conda.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tljh/conda.py b/tljh/conda.py index 02e28cc..96ef324 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -8,6 +8,7 @@ import hashlib import contextlib import tempfile import urllib.request +from distutils.version import LooseVersion as V def md5_file(fname): @@ -28,12 +29,13 @@ def check_miniconda_version(prefix, version): Return true if a miniconda install with version exists at prefix """ try: - return subprocess.check_output([ + installed_version = subprocess.check_output([ os.path.join(prefix, 'bin', 'conda'), '-V' - ]).decode().strip() == 'conda {}'.format(version) + ]).decode().strip().split()[1] + return V(installed_version) >= V(version) except (subprocess.CalledProcessError, FileNotFoundError): - # Conda doesn't exist, or wrong version + # Conda doesn't exist return False From e97b81fe7ac6fb40bac6ab2c1850dd7ba296a9ba Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:30:11 -0700 Subject: [PATCH 5/9] Require miniconda to be installed for conda. to work We can no longer assume that sys.executable has conda installed. Instead, we require that prefix has conda installed. This requires miniconda to be installed into prefix. --- tests/test_conda.py | 33 ++++++--------------------------- tljh/conda.py | 19 ------------------- 2 files changed, 6 insertions(+), 46 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index 7b372e5..4f3f608 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -8,42 +8,23 @@ import subprocess import tempfile -@pytest.fixture +@pytest.fixture(scope='module') def prefix(): """ - Provide a temporary directory to make environments in. + Provide a temporary directory with a conda environment """ + miniconda_version = '4.5.4' + miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" with tempfile.TemporaryDirectory() as tmpdir: + with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: + conda.install_miniconda(installer_path, tmpdir) yield tmpdir -def test_create_environment(prefix): - """ - Test conda environment creation - - An empty conda environment doesn't seem to have anything in it, - so we just check for directory existence. - """ - conda.ensure_conda_env(prefix) - assert os.path.exists(prefix) - - -def test_ensure_environment(prefix): - """ - Test second call to ensure_conda_env works as expected - - A conda environment already exists, so we it should just do nothing - """ - conda.ensure_conda_env(prefix) - assert os.path.exists(prefix) - conda.ensure_conda_env(prefix) - - def test_ensure_packages(prefix): """ Test installing packages in conda environment """ - conda.ensure_conda_env(prefix) conda.ensure_conda_packages(prefix, ['numpy']) # Throws an error if this fails subprocess.check_call([ @@ -57,7 +38,6 @@ def test_ensure_pip_packages(prefix): """ Test installing pip packages in conda environment """ - conda.ensure_conda_env(prefix) conda.ensure_conda_packages(prefix, ['pip']) conda.ensure_pip_packages(prefix, ['numpy']) # Throws an error if this fails @@ -72,7 +52,6 @@ def test_ensure_pip_requirements(prefix): """ Test installing pip packages with requirements.txt in conda environment """ - conda.ensure_conda_env(prefix) conda.ensure_conda_packages(prefix, ['pip']) with tempfile.NamedTemporaryFile() as f: # Sample small package to test diff --git a/tljh/conda.py b/tljh/conda.py index 96ef324..d79de1e 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -77,25 +77,6 @@ def install_miniconda(installer_path, prefix): subprocess.check_call(["chmod", "-R", "o-w", prefix]) -def ensure_conda_env(prefix): - """ - Ensure a conda environment in the prefix - """ - conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] - abspath = os.path.abspath(prefix) - try: - output = json.loads( - subprocess.check_output(conda_executable + ['create', '--json', '--prefix', abspath]).decode() - ) - except subprocess.CalledProcessError as e: - output = json.loads(e.output.decode()) - if 'error' in output and output['error'] == f'CondaValueError: prefix already exists: {abspath}': - return - raise - if 'success' in output and output['success'] == True: - return - - def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. From 376489cc913a41d0d073350dfc4706282142331a Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:36:51 -0700 Subject: [PATCH 6/9] Use ubuntu 18.04 image for running unit tests This matches the image in which we run integration tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c75ef6..0c0ab35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: unit-test: docker: - - image: continuumio/miniconda3:4.5.4 + - image: ubuntu:18.04 working_directory: ~/repo From b8b2dcf766b95e319e3a6ce64c14a28f54ce3cf9 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:42:15 -0700 Subject: [PATCH 7/9] Install python manually in the unit test container --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c0ab35..48aeb95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2 jobs: unit-test: docker: + # Match target OS of TLJH - image: ubuntu:18.04 working_directory: ~/repo @@ -9,6 +10,12 @@ jobs: steps: - checkout + # Setup Python + - run: + name: install python + command: | + apt-get update --yes && apt-get install --yes python3 python3-pip + # Download and cache dependencies - restore_cache: keys: From 40c7cc84b88331b7b26bd1452162d0049140990a Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:47:04 -0700 Subject: [PATCH 8/9] Use pip3 instead of 'pip' when unit testing Base system pip in ubuntu does not respond to 'pip3' --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 48aeb95..27157c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,8 +25,8 @@ jobs: - run: name: install dependencies command: | - pip install -r dev-requirements.txt - pip install -e . + pip3 install -r dev-requirements.txt + pip3 install -e . - save_cache: paths: From 88b291d989f530069fe1507b1d8e79f2bc562068 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 20 Jul 2018 11:49:34 -0700 Subject: [PATCH 9/9] Cache /usr/local/lib/python3.6 when using pip --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27157c5..82ffd8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,8 +19,8 @@ jobs: # Download and cache dependencies - restore_cache: keys: - - v1-dependencies-miniconda3-4.5.4-{{ checksum "setup.py" }}-{{ checksum "dev-requirements.txt" }} - - v1-dependencies-miniconda3-4.5.4- + - v1-dependencies-py3.6-{{ checksum "setup.py" }}-{{ checksum "dev-requirements.txt" }} + - v1-dependencies-py3.6- - run: name: install dependencies @@ -30,8 +30,8 @@ jobs: - save_cache: paths: - - /opt/conda - key: v1-dependencies-miniconda3-4.5.4-{{ checksum "setup.py" }}-{{ checksum "dev-requirements.txt" }} + - /usr/local/lib/python3.6 + key: v1-dependencies-py3.6-{{ checksum "setup.py" }}-{{ checksum "dev-requirements.txt" }} - run: name: run unit tests