zhangyiss/the-littlest-jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/the-littlest-jupyterhub.git synced 2025-12-18 21:54:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use venv for base hub environment

Browse Source 
- TLJH should support raspberry pi, which runs ARM. conda does
  not support ARM.
- Get nodejs from nodesource instead of conda or default repositories.
  Default repositories get out of date pretty quickly.
- Install CHP from npm
This commit is contained in:
yuvipanda
2018-07-19 17:30:09 -07:00
parent 5007911d59
commit fc0ecb6699
6 changed files with 227 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
bootstrap/bootstrap.py
View File

@@ -11,131 +11,45 @@ Constraints:
- Be compatible with Python 3.4 (since we support Ubuntu 16.04)
- Use stdlib modules only
"""
from distutils.version import LooseVersion as V
import os
import subprocess
import urllib.request
import contextlib
import hashlib
import tempfile
import sys
def md5_file(fname):
"""
Return md5 of a given filename
Copied from https://stackoverflow.com/a/3431838
"""
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def check_miniconda_version(prefix, version):
"""
Return true if a miniconda install with version exists at prefix
"""
try:
installed_version = subprocess.check_output([
os.path.join(prefix, 'bin', 'conda'),
'-V'
]).decode().strip().split()[1]
except (subprocess.CalledProcessError, FileNotFoundError):
# Conda doesn't exist, or wrong version
return False
else:
return V(installed_version) >= V(version)
@contextlib.contextmanager
def download_miniconda_installer(version, md5sum):
"""
Context manager to download miniconda installer of given version.
This should be used as a contextmanager. It downloads miniconda installer
of given version, verifies the md5sum & provides path to it to the `with`
block to run.
"""
with tempfile.NamedTemporaryFile() as f:
installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version)
urllib.request.urlretrieve(installer_url, f.name)
if md5_file(f.name) != md5sum:
raise Exception('md5 hash mismatch! Downloaded file corrupted')
yield f.name
def install_miniconda(installer_path, prefix):
"""
Install miniconda with installer at installer_path under prefix
"""
subprocess.check_output([
'/bin/bash',
installer_path,
'-u', '-b',
'-p', prefix
], stderr=subprocess.STDOUT)
# fix permissions on initial install
# a few files have the wrong ownership and permissions initially
# when the installer is run as root
subprocess.check_call(
["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix]
)
subprocess.check_call(["chmod", "-R", "o-w", prefix])
def pip_install(prefix, packages, editable=False):
"""
Install pip packages in the conda environment under prefix.
Packages are upgraded if possible.
Set editable=True to add '--editable' to the pip install commandline.
Very useful when doing active development
"""
flags = ['--upgrade']
if editable:
flags.append('--editable')
subprocess.check_output([
os.path.join(prefix, 'bin', 'python3'),
'-m', 'pip',
'install',
] + flags + packages)
def main():
install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
hub_prefix = os.path.join(install_prefix, 'hub')
miniconda_version = '4.5.4'
miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d"
print('Checking if TLJH is already installed...')
if not check_miniconda_version(hub_prefix, miniconda_version):
initial_setup = True
print('Downloading & setting up hub environment...')
with download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path:
install_miniconda(installer_path, hub_prefix)
print('Hub environment set up!')
else:
if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')):
print('TLJH already installed, upgrading...')
initial_setup = False
print('TLJH is already installed, will try to upgrade')
else:
print('Setting up hub environment')
initial_setup = True
subprocess.check_output(['apt-get', 'update', '--yes'])
# gnupg2 is needed for nodesource to work
subprocess.check_output(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'gnupg2'])
os.makedirs(hub_prefix, exist_ok=True)
subprocess.check_output(['python3', '-m', 'venv', hub_prefix])
if initial_setup:
print('Setting up TLJH installer...')
else:
print('Upgrading TLJH installer...')
is_dev = os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes'
pip_flags = ['--upgrade']
if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes':
pip_flags.append('--editable')
tljh_repo_path = os.environ.get(
'TLJH_BOOTSTRAP_PIP_SPEC',
'git+https://github.com/yuvipanda/the-littlest-jupyterhub.git'
)
pip_install(hub_prefix, [tljh_repo_path], editable=is_dev)
subprocess.check_output([
os.path.join(hub_prefix, 'bin', 'pip'),
'install'
] + pip_flags + [tljh_repo_path])
print('Starting TLJH installer...')
os.execv(

10
tests/test_installer.py Normal file
View File

@@ -0,0 +1,10 @@
"""
Unit test functions in installer.py
"""
from tljh import installer
import os
def test_ensure_node():
installer.ensure_node()
assert os.path.exists('/usr/bin/node')

43
tljh/apt.py Normal file
View File

@@ -0,0 +1,43 @@
"""
Utilities for working with the apt package manager
"""
import os
import subprocess
def trust_gpg_key(key):
"""
Trust given GPG public key.
key is a GPG public key (bytes) that can be passed to apt-key add via stdin.
"""
subprocess.run(['apt-key', 'add', '-'], input=key, check=True)
def add_source(name, source_url, section):
"""
Add a debian package source.
distro is determined from /etc/os-release
"""
# lsb_release is not installed in most docker images by default
distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}']).decode().strip()
line = f'deb {source_url} {distro} {section}'
with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f:
# Write out deb line only if it already doesn't exist
if f.read() != line:
f.seek(0)
f.write(line)
f.truncate()
subprocess.check_output(['apt-get', 'update', '--yes'])
def install_packages(packages):
"""
Install debian packages
"""
subprocess.check_output([
'apt-get',
'install',
'--yes'
] + packages)

77
tljh/conda.py
View File

@@ -4,20 +4,86 @@ Wrap conda commandline program
import os
import subprocess
import json
import sys
import hashlib
import contextlib
import tempfile
import urllib.request
# Use sys.executable to call conda to avoid needing to fudge PATH
CONDA_EXECUTABLE = [sys.executable, '-m', 'conda']
def md5_file(fname):
"""
Return md5 of a given filename
Copied from https://stackoverflow.com/a/3431838
"""
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def check_miniconda_version(prefix, version):
"""
Return true if a miniconda install with version exists at prefix
"""
try:
return subprocess.check_output([
os.path.join(prefix, 'bin', 'conda'),
'-V'
]).decode().strip() == 'conda {}'.format(version)
except (subprocess.CalledProcessError, FileNotFoundError):
# Conda doesn't exist, or wrong version
return False
@contextlib.contextmanager
def download_miniconda_installer(version, md5sum):
"""
Context manager to download miniconda installer of given version.
This should be used as a contextmanager. It downloads miniconda installer
of given version, verifies the md5sum & provides path to it to the `with`
block to run.
"""
with tempfile.NamedTemporaryFile() as f:
installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version)
urllib.request.urlretrieve(installer_url, f.name)
if md5_file(f.name) != md5sum:
raise Exception('md5 hash mismatch! Downloaded file corrupted')
yield f.name
def install_miniconda(installer_path, prefix):
"""
Install miniconda with installer at installer_path under prefix
"""
subprocess.check_output([
'/bin/bash',
installer_path,
'-u', '-b',
'-p', prefix
], stderr=subprocess.STDOUT)
# fix permissions on initial install
# a few files have the wrong ownership and permissions initially
# when the installer is run as root
subprocess.check_call(
["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix]
)
subprocess.check_call(["chmod", "-R", "o-w", prefix])
def ensure_conda_env(prefix):
"""
Ensure a conda environment in the prefix
"""
conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda']
abspath = os.path.abspath(prefix)
try:
output = json.loads(
subprocess.check_output(CONDA_EXECUTABLE + ['create', '--json', '--prefix', abspath]).decode()
subprocess.check_output(conda_executable + ['create', '--json', '--prefix', abspath]).decode()
)
except subprocess.CalledProcessError as e:
output = json.loads(e.output.decode())
@@ -32,10 +98,11 @@ def ensure_conda_packages(prefix, packages):
"""
Ensure packages (from conda-forge) are installed in the conda prefix.
"""
conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda']
abspath = os.path.abspath(prefix)
# Let subprocess errors propagate
# FIXME: raise different exception when using
raw_output = subprocess.check_output(CONDA_EXECUTABLE + [
raw_output = subprocess.check_output(conda_executable + [
'install',
'-c', 'conda-forge', # Make customizable if we ever need to
'--json',

90
tljh/installer.py
View File

@@ -9,7 +9,7 @@ from urllib.request import urlopen, URLError
from ruamel.yaml import YAML
from tljh import conda, systemd, user
from tljh import conda, systemd, user, apt
INSTALL_PREFIX = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'hub')
@@ -21,6 +21,79 @@ HERE = os.path.abspath(os.path.dirname(__file__))
rt_yaml = YAML()
def ensure_node():
"""
Ensure nodejs from nodesource is installed
"""
key = b"""
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1
Comment: GPGTools - https://gpgtools.org
mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+
W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs
fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy
qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7
89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD
Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7
C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/
G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc
Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft
qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm
EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB
tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2
AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy
jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ
kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1
GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm
XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU
VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka
1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI
IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc
pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn
xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y
gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI
AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ
fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ
Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh
41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea
JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD
xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi
vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/
aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o
QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK
yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3
QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek
fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK
CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec
0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1
PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh
qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15
ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac
hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb
DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4
xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu
G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd
sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy
/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g==
=CLGF
-----END PGP PUBLIC KEY BLOCK-----
""".strip()
apt.trust_gpg_key(key)
apt.add_source('nodesource', f'https://deb.nodesource.com/node_10.x', 'main')
apt.install_packages(['nodejs'])
def ensure_chp_package(prefix):
"""
Ensure CHP is installed
"""
if not os.path.exists(os.path.join(prefix, 'node_modules', '.bin', 'configurable-http-proxy')):
subprocess.check_call([
'npm', 'install', 'configurable-http-proxy@3.1.0'
], cwd=prefix)
def ensure_jupyterhub_service(prefix):
"""
Ensure JupyterHub & CHP Services are set up properly
@@ -70,7 +143,6 @@ def ensure_jupyterhub_package(prefix):
hub environment be installed with pip prevents accidental mixing of python
and conda packages!
"""
conda.ensure_conda_packages(prefix, ['configurable-http-proxy==3.1.0'])
conda.ensure_pip_packages(prefix, [
'jupyterhub==0.9.1',
'jupyterhub-dummyauthenticator==0.3.1',
@@ -103,7 +175,14 @@ def ensure_user_environment(user_requirements_txt_file):
Set up user conda environment with required packages
"""
print("Setting up user environment...")
conda.ensure_conda_env(USER_ENV_PREFIX)
miniconda_version = '4.5.4'
miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d"
if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version):
print('Downloading & setting up user environment...')
with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path:
conda.install_miniconda(installer_path, USER_ENV_PREFIX)
conda.ensure_conda_packages(USER_ENV_PREFIX, [
# Conda's latest version is on conda much more so than on PyPI.
'conda==4.5.8'
@@ -194,12 +273,11 @@ def main():
ensure_usergroups()
ensure_user_environment(args.user_requirements_txt_url)
# Weird setuptools issue creates a few world-writable metadata files.
# Fix it:
subprocess.check_call(["chmod", "-R", "o-w", os.path.join(HUB_ENV_PREFIX, "pkgs")])
print("Setting up JupyterHub...")
ensure_node()
ensure_jupyterhub_package(HUB_ENV_PREFIX)
ensure_chp_package(HUB_ENV_PREFIX)
ensure_jupyterhub_service(HUB_ENV_PREFIX)
ensure_jupyterhub_running()

4
tljh/systemd-units/configurable-http-proxy.service
View File

@@ -15,9 +15,7 @@ PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
EnvironmentFile={install_prefix}/state/configurable-http-proxy.secret
# Set PATH so env can find correct node
Environment=PATH=$PATH:{install_prefix}/hub/bin
ExecStart={install_prefix}/hub/bin/configurable-http-proxy \
ExecStart={install_prefix}/hub/node_modules/.bin/configurable-http-proxy \
--ip 0.0.0.0 \
--port 80 \
--api-ip 127.0.0.1 \