From f4831f051f9f0a54a0c499a05633ac80c93aebc2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sat, 18 May 2019 13:43:34 -0700 Subject: [PATCH 1/8] Use c.Spawner to set mem_limit & cpu_limit mem_limit & cpu_limit are traitlets on the parent Spawner class. Setting these here allows plugins to do the dangerous job of swapping the SystemdSpawner out for something else --- tests/test_configurer.py | 4 ++-- tljh/configurer.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_configurer.py b/tests/test_configurer.py index 98dd106..641e407 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -66,7 +66,7 @@ def test_default_memory_limit(): Test default per user memory limit """ c = apply_mock_config({}) - assert c.SystemdSpawner.mem_limit is None + assert c.Spawner.mem_limit is None def test_set_memory_limit(): @@ -74,7 +74,7 @@ def test_set_memory_limit(): Test setting per user memory limit """ c = apply_mock_config({'limits': {'memory': '42G'}}) - assert c.SystemdSpawner.mem_limit == '42G' + assert c.Spawner.mem_limit == '42G' def test_app_default(): diff --git a/tljh/configurer.py b/tljh/configurer.py index 491a5f8..e909ac7 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -162,8 +162,8 @@ def update_limits(c, config): """ limits = config['limits'] - c.SystemdSpawner.mem_limit = limits['memory'] - c.SystemdSpawner.cpu_limit = limits['cpu'] + c.Spawner.mem_limit = limits['memory'] + c.Spawner.cpu_limit = limits['cpu'] def update_user_environment(c, config): From 12e984febee7c6ca7a23ffe3b63683116f8ec0f3 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sat, 18 May 2019 14:06:16 -0700 Subject: [PATCH 2/8] Add note about not running on your own laptop Fixes #210 --- docs/install/custom-server.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/install/custom-server.rst b/docs/install/custom-server.rst index 651abce..2d3dc65 100644 --- a/docs/install/custom-server.rst +++ b/docs/install/custom-server.rst @@ -4,11 +4,15 @@ Installing on your own server ============================= -.. note:: - You should use this if your cloud provider does not already have a direct tutorial, - or if you have experience setting up servers. +Follow this guide if your cloud provider doesn't have a direct tutorial, or +you are setting this up on a bare metal server. +.. warning:: + + Do **not** install TLJH directly on your laptop or personal computer! + It will most likely open up exploitable security holes when run directly + on your personal computer. Goal ==== From f5c8b7b86cecfde287c2078202e7c50ac966f4eb Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 19 May 2019 11:05:24 -0700 Subject: [PATCH 3/8] Explicitlly document that we don't support running inside docker --- docs/contributing/dev-setup.rst | 2 +- docs/install/custom-server.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst index e69ca68..af0a3b8 100644 --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -1,4 +1,4 @@ -.. _contributing_dev_setup: +.. _contributing/dev-setup: ================================== Setting up Development Environment diff --git a/docs/install/custom-server.rst b/docs/install/custom-server.rst index 2d3dc65..e31c492 100644 --- a/docs/install/custom-server.rst +++ b/docs/install/custom-server.rst @@ -14,6 +14,12 @@ you are setting this up on a bare metal server. It will most likely open up exploitable security holes when run directly on your personal computer. +.. note:: + + Running TLJH *inside* a docker container is not supported, since we depend + on systemd. If you want to run TLJH locally for development, see + :ref:`contributing/dev-setup`. + Goal ==== From 536e435c06ddba628c85369c73d9683cdb8ac802 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 19 May 2019 22:29:18 -0700 Subject: [PATCH 4/8] Retry downloading traefik if it fails Fixes #314 --- setup.py | 1 + tljh/traefik.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 61f8ead..4597a95 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ setup( 'jinja2', 'pluggy>0.7<1.0', 'passlib', + 'backoff', 'jupyterhub-traefik-proxy==0.1.*' ], entry_points={ diff --git a/tljh/traefik.py b/tljh/traefik.py index 4581953..4e05b7a 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -1,10 +1,11 @@ """Traefik installation and setup""" import hashlib import os -from urllib.request import urlretrieve +from urllib.request import urlretrieve, ContentTooShortError from jinja2 import Template from passlib.apache import HtpasswdFile +import backoff from tljh.configurer import load_config @@ -26,7 +27,12 @@ def checksum_file(path): hasher.update(chunk) return hasher.hexdigest() - +@backoff.on_exception( + backoff.expo, + # Retry when connection is reset or we think we didn't download entire file + (ConnectionResetError, ContentTooShortError), + max_tries=2 +) def ensure_traefik_binary(prefix): """Download and install the traefik binary""" traefik_bin = os.path.join(prefix, "bin", "traefik") From 9f2a04ac4918983ca6dd246d6e82329f5cc0c1f2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 19 May 2019 22:44:49 -0700 Subject: [PATCH 5/8] Add more validation to bootstrap.py - Check for Python Version - Check if systemd is present - Provide more useful error message when running inside an unprepared docker container Ref #16 --- bootstrap/bootstrap.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 3a42fb2..adbc9fc 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -16,6 +16,7 @@ import os import subprocess import sys import logging +import shutil def get_os_release_variable(key): @@ -31,8 +32,10 @@ def get_os_release_variable(key): "source /etc/os-release && echo ${{{key}}}".format(key=key) ]).decode().strip() -def main(): - +def validate_host(): + """ + Make sure TLJH is installable in current host + """ # Support only Ubuntu 18.04+ distro = get_os_release_variable('ID') version = float(get_os_release_variable('VERSION_ID')) @@ -43,6 +46,20 @@ def main(): print('The Littlest JupyterHub requires Ubuntu 18.04 or higher') sys.exit(1) + if sys.version_info < (3, 5): + print("bootstrap.py must be run with at least Python 3.5") + sys.exit(1) + + if not (shutil.which('systemd') and shutil.which('systemctl')): + print("Systemd is required to run TLJH") + # Only fail running inside docker if systemd isn't present + if os.path.exists('/.dockerenv'): + print("Running inside a plain docker container isn't supported") + print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") + sys.exit(1) + +def main(): + validate_host() install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') From 0370b7513e301884bae2ab924897a6245c6acfed Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 19 May 2019 23:00:43 -0700 Subject: [PATCH 6/8] Add test for failure when running inside a plain docker container --- integration-tests/test_bootstrap.py | 33 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 5bf143f..a93fc5a 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -2,14 +2,9 @@ Test running bootstrap script in different circumstances """ import subprocess +from textwrap import dedent - -def test_ubuntu_too_old(): - """ - Error with a useful message when running in older Ubuntu - """ - container_name = 'old-distro-test' - +def run_bootstrap(container_name, image): # stop container if it is already running subprocess.run([ 'docker', 'rm', '-f', container_name @@ -17,7 +12,7 @@ def test_ubuntu_too_old(): # Start a detached Ubuntu 16.04 container subprocess.check_call([ - 'docker', 'run', '--detach', '--name', container_name, 'ubuntu:16.04', + 'docker', 'run', '--detach', '--name', container_name, image, '/bin/bash', '-c', 'sleep 1000s' ]) # Install python3 inside the ubuntu container @@ -35,10 +30,26 @@ def test_ubuntu_too_old(): 'bootstrap/', f'{container_name}:/srv' ]) - # Run bootstrap script, validate that it fails appropriately - output = subprocess.run([ + # Run bootstrap script, return the output + return subprocess.run([ 'docker', 'exec', '-i', container_name, 'python3', '/srv/bootstrap/bootstrap.py' ], check=False, stdout=subprocess.PIPE, encoding='utf-8') + +def test_ubuntu_too_old(): + """ + Error with a useful message when running in older Ubuntu + """ + output = run_bootstrap('old-distro-test', 'ubuntu:16.04') assert output.stdout == 'The Littlest JupyterHub requires Ubuntu 18.04 or higher\n' - assert output.returncode == 1 \ No newline at end of file + assert output.returncode == 1 + + +def test_inside_plain_docker(): + output = run_bootstrap('plain-docker-test', 'ubuntu:18.04') + assert output.stdout.strip() == dedent(""" + Systemd is required to run TLJH + Running inside a plain docker container isn't supported + For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html + """).strip() + assert output.returncode == 1 From aab8cf2dc04e00f3dee7bb895141aa3ba09a0731 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Mon, 20 May 2019 08:33:03 -0700 Subject: [PATCH 7/8] Remove stray .DS_Store files Add it to .gitignore so we prevent this from happening again --- .gitignore | 3 +++ docs/.DS_Store | Bin 6148 -> 0 bytes docs/images/.DS_Store | Bin 6148 -> 0 bytes docs/images/providers/.DS_Store | Bin 6148 -> 0 bytes docs/troubleshooting/.DS_Store | Bin 6148 -> 0 bytes 5 files changed, 3 insertions(+) delete mode 100644 docs/.DS_Store delete mode 100644 docs/images/.DS_Store delete mode 100644 docs/images/providers/.DS_Store delete mode 100644 docs/troubleshooting/.DS_Store diff --git a/.gitignore b/.gitignore index 894a44c..1e5c982 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ venv.bak/ # mypy .mypy_cache/ + +# OS X stuff +.DS_Store diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index 1286e278017668d42d6262b4e5040900540ec737..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK+e!m55Ixb>78LBGkNE}vU@7$r`~bCEsfD!>x5d}}bkAH`y1Mu%A~TSjBy-8+ zz$QBY(tN+V1!e%IY=TJ!10u_Vsxvp95R=B(VS{?r*3GgE1p13hlKTV?XtBc++cE#r zNV-?3QLv@rovJ1FYO=-J4rxYXywK=_s&cw~L627kUE>+;K|!9l--aP=#|F)*YAiwU z_BD-bO`vf5B^n8*JC>njB{vN=hSX4!xf0Dad5MwCo$J}{D#y;y+>yNaNWSx-cuBfD z<7YKTstltK27-Y;1LroK$@_oDPiC;l@4Lh-7zhUb83SpwxL?e9D1TeO9Z%lc%=XA8 rB6gh&C>-~G0yvR-E5Sgk0v*zgTNe*1+{E1~sOglYfP!X7f>erxSb!iJmTV2K0gY{w z{72CKP=kGN}82ucKnYF(B8Q)#ywLogFE|Ijgvwp*n6Ffi@d6R|4MW=y3d}!a9p?N z_Flf~9YrJ6M0H$Gt1zA%(9_qr9wpY^Q>wVNS5ZChTw?e5N6)Q6^`%PzOHoITKPKPnZ8{Ke)F^4f`fg%=c7`_mkM;()b@k{}7wT5I# z&`aAvEQ7IVa{NUG@ZFV|&n`i)+35Rw5882*X4UE&FH9AuOEZ>jm+kWGT=~Rn$)1<_ zSvPI?og@6U=Vz_J*!xn(pSB+yM9to*v#=$T%#V_0M8QhU_L`dhBWE zVH9R9wWx7emTlRm&SJm6R;yRUYJGiB5&c@NS`q6T)xp4dvaO}%wawi?-rU~ZKRiA? zzi6^Ke3we*4G!TIjFEGP>3JN$zA<5B;>znJu}m(&KU{3M2q7^*3=jho%7D4ftn!5J zpSDd55CcEQ0NxJ*6wx)9YgAVU45|eHEP`7J*w{;8jxgvN%r(Lc2-m5AI+dFigX?tg z3lrxW%r)wC#!d6V&6Bz5P`G+JtS?kJ7eSSPU!Vc#8q76<2Za6zC>m%W I27Z-+54Le%_y7O^ diff --git a/docs/troubleshooting/.DS_Store b/docs/troubleshooting/.DS_Store deleted file mode 100644 index 4f10a9cddcb4db18676f06bc9de2279888fec0f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyG{c^3>-s>Aexkv`wRTRDoVb9ABe(HAVoSv1oc&X7oV2#Lx|`?gG7VIl0CaV z&#iWf^BI7x--df&1z<^c#D|Bu`E&P~T~)@2biU&m1D^1T<8Yi+Urspp1~24{c>m6S zJnYBuaG9j66p#W^Knh3!De!9ry!X Date: Mon, 20 May 2019 09:52:50 -0700 Subject: [PATCH 8/8] Say 'running inside a docker container', not 'plain docker' 'Plain docker' makes no sense --- bootstrap/bootstrap.py | 3 ++- integration-tests/test_bootstrap.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index adbc9fc..70bdbf8 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -54,7 +54,8 @@ def validate_host(): print("Systemd is required to run TLJH") # Only fail running inside docker if systemd isn't present if os.path.exists('/.dockerenv'): - print("Running inside a plain docker container isn't supported") + print("Running inside a docker container without systemd isn't supported") + print("We recommend against running a production TLJH instance inside a docker container") print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") sys.exit(1) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index a93fc5a..fa02b68 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -45,11 +45,12 @@ def test_ubuntu_too_old(): assert output.returncode == 1 -def test_inside_plain_docker(): +def test_inside_no_systemd_docker(): output = run_bootstrap('plain-docker-test', 'ubuntu:18.04') assert output.stdout.strip() == dedent(""" Systemd is required to run TLJH - Running inside a plain docker container isn't supported + Running inside a docker container without systemd isn't supported + We recommend against running a production TLJH instance inside a docker container For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html """).strip() assert output.returncode == 1