From 972b15b95a17f5d04652f4991c690556fca39937 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jul 2020 22:59:56 +0300 Subject: [PATCH 01/21] Serve a temporary html page while tljh is building --- bootstrap/bootstrap.py | 35 +++++++++++++++++++++- bootstrap/index.html | 67 ++++++++++++++++++++++++++++++++++++++++++ tljh/installer.py | 13 ++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 bootstrap/index.html diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 434ea35..3eff96c 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -13,10 +13,13 @@ Constraints: - Use stdlib modules only """ import os +from http.server import SimpleHTTPRequestHandler, HTTPServer +import multiprocessing import subprocess import sys import logging import shutil +import urllib.request logger = logging.getLogger(__name__) @@ -90,7 +93,38 @@ def validate_host(): print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") sys.exit(1) +class LoaderPageRequestHandler(SimpleHTTPRequestHandler): + def do_GET(self): + if self.path == "/logs": + with open("/opt/tljh/installer.log", "rb") as log_file: + content = log_file.read() + self.wfile.write(content) + else: + self.path = '/index.html' + return SimpleHTTPRequestHandler.do_GET(self) + +def serve_forever(server): + try: + server.serve_forever() + except KeyboardInterrupt: + pass + def main(): + # Serve the loading page until TLJH builds + url="https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/index.html" + urllib.request.urlretrieve(url, "index.html") + + # If the bootstrap is run to upgrade TLJH, then this will raise an "Address already in use" error + try: + loading_page_server = HTTPServer(("", 80), LoaderPageRequestHandler) + p = multiprocessing.Process(target=serve_forever, args=(loading_page_server,)) + p.start() + with open('/loading.pid', 'w+') as f: + f.write(str(p.pid)) + except OSError: + # Only serve the loading page when installing TLJH + pass + validate_host() install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') @@ -178,6 +212,5 @@ def main(): ] + sys.argv[1:] ) - if __name__ == '__main__': main() diff --git a/bootstrap/index.html b/bootstrap/index.html new file mode 100644 index 0000000..0279993 --- /dev/null +++ b/bootstrap/index.html @@ -0,0 +1,67 @@ + + + + + + +
+
Please wait while your TLJH is building...
+
Click the button below to see the logs
+ + + + + diff --git a/tljh/installer.py b/tljh/installer.py index b18e2ac..77112d0 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -6,6 +6,7 @@ import itertools import logging import os import secrets +import signal import subprocess import sys import time @@ -499,6 +500,18 @@ def main(): ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) ensure_jupyterlab_extensions() + + # Stop the http server with the loading page before traefik starts + try: + with open(pidfile, "r") as f: + pid = f.read() + os.kill(int(pid), signal.SIGINT) + # Remove the pid file and the temporary html page + os.remove('/loading.pid') + os.remove('/index.html') + except: + pass + ensure_jupyterhub_service(HUB_ENV_PREFIX) ensure_jupyterhub_running() ensure_symlinks(HUB_ENV_PREFIX) From 3aa8255110b5649bbb896a2b25f94f8c2dab2d82 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 3 Aug 2020 17:58:48 +0300 Subject: [PATCH 02/21] Center divs --- bootstrap/index.html | 60 ++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/bootstrap/index.html b/bootstrap/index.html index 0279993..6b37cd1 100644 --- a/bootstrap/index.html +++ b/bootstrap/index.html @@ -1,13 +1,16 @@ - - + + -
-
Please wait while your TLJH is building...
-
Click the button below to see the logs
- +
+
Please wait while your TLJH is building...
+
Click the button below to see the logs
+ + From 56f9db5130bf55a943e8d9872d16fa38a58ffdc7 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 3 Aug 2020 18:38:39 +0300 Subject: [PATCH 03/21] Refresh the main page every 30s --- bootstrap/index.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bootstrap/index.html b/bootstrap/index.html index 6b37cd1..3220733 100644 --- a/bootstrap/index.html +++ b/bootstrap/index.html @@ -1,16 +1,15 @@ +
Please wait while your TLJH is building...
Click the button below to see the logs
+
Tip: to update the logs, refresh the page
- + + + +""" + logger = logging.getLogger(__name__) def get_os_release_variable(key): @@ -99,7 +180,11 @@ class LoaderPageRequestHandler(SimpleHTTPRequestHandler): with open("/opt/tljh/installer.log", "rb") as log_file: content = log_file.read() self.wfile.write(content) - elif self.path == "/index.html" or self.path == "/favicon.ico": + elif self.path == "/index.html": + self.path = "/var/run/index.html" + return SimpleHTTPRequestHandler.do_GET(self) + elif self.path == "/favicon.ico": + self.path = "/var/run/favicon.ico" return SimpleHTTPRequestHandler.do_GET(self) elif self.path == "/": self.send_response(302) @@ -118,10 +203,10 @@ def main(): temp_page_flag = "--temporary-page" if temp_page_flag in sys.argv: # Serve the loading page until TLJH builds - index_url="https://raw.githubusercontent.com/GeorgianaElena/the-littlest-jupyterhub/in-progress-page/bootstrap/index.html" + with open("/var/run/index.html", "w+") as f: + f.write(html) favicon_url="https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico" - urllib.request.urlretrieve(index_url, "index.html") - urllib.request.urlretrieve(favicon_url, "favicon.ico") + urllib.request.urlretrieve(favicon_url, "/var/run/favicon.ico") # If the bootstrap is run to upgrade TLJH, then this will raise an "Address already in use" error try: diff --git a/bootstrap/index.html b/bootstrap/index.html deleted file mode 100644 index 3220733..0000000 --- a/bootstrap/index.html +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - -
-
Please wait while your TLJH is building...
-
Click the button below to see the logs
-
Tip: to update the logs, refresh the page
- - - - - From cef898fd318e03e8818dc057b6cbdd9819586c5b Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 17 Aug 2020 16:47:55 +0300 Subject: [PATCH 08/21] Rename flags and work with sys.argv copy --- bootstrap/bootstrap.py | 16 ++++++++++------ tljh/installer.py | 7 +++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 6e08fd9..0b3e688 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -30,7 +30,7 @@ html = """ - +
Please wait while your TLJH is building...
Click the button below to see the logs
@@ -200,8 +200,10 @@ def serve_forever(server): pass def main(): - temp_page_flag = "--temporary-page" - if temp_page_flag in sys.argv: + flags = sys.argv[1:] + temp_page_flag = "--show-progress-page" + + if temp_page_flag in flags: # Serve the loading page until TLJH builds with open("/var/run/index.html", "w+") as f: f.write(html) @@ -213,8 +215,10 @@ def main(): loading_page_server = HTTPServer(("", 80), LoaderPageRequestHandler) p = multiprocessing.Process(target=serve_forever, args=(loading_page_server,)) p.start() - # Put the pid after the temp page flag to be passed to the istaller - sys.argv.insert(sys.argv.index(temp_page_flag)+1, str(p.pid)) + # Pass the server's pid as a flag to the istaller + flags.remove(temp_page_flag) + pid_flag = "--progress-page-server-pid" + flags.extend([pid_flag, str(p.pid)]) except OSError: # Only serve the loading page when installing TLJH pass @@ -303,7 +307,7 @@ def main(): os.path.join(hub_prefix, 'bin', 'python3'), '-m', 'tljh.installer', - ] + sys.argv[1:] + ] + flags ) if __name__ == '__main__': diff --git a/tljh/installer.py b/tljh/installer.py index 7756757..0a21967 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -487,8 +487,9 @@ def main(): help='Plugin pip-specs to install' ) argparser.add_argument( - '--temporary-page', - help='Serve a temporary page while TLJH is building' + '--progress-page-server-pid', + type=int, + help='The pid of the progress page server' ) args = argparser.parse_args() @@ -510,8 +511,6 @@ def main(): try: os.kill(int(args.temporary_page), signal.SIGINT) # Remove the pid file and the temporary html page - os.remove('/index.html') - os.remove('/favicon.ico') except Exception as e: pass From 1ce172ae3e80ec8afdc85ce1fdf37f08af037f70 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 17 Aug 2020 19:04:25 +0300 Subject: [PATCH 09/21] Added more comments --- bootstrap/bootstrap.py | 20 +++++++++++++++----- tljh/installer.py | 5 ++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 0b3e688..55cfbc0 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -177,9 +177,13 @@ def validate_host(): class LoaderPageRequestHandler(SimpleHTTPRequestHandler): def do_GET(self): if self.path == "/logs": - with open("/opt/tljh/installer.log", "rb") as log_file: - content = log_file.read() - self.wfile.write(content) + with open("/opt/tljh/installer.log", "r") as log_file: + logs = log_file.read() + + self.send_response(200) + self.send_header('Content-Type', 'text/plain; charset=utf-8') + self.end_headers() + self.wfile.write(logs.encode('utf-8')) elif self.path == "/index.html": self.path = "/var/run/index.html" return SimpleHTTPRequestHandler.do_GET(self) @@ -203,8 +207,10 @@ def main(): flags = sys.argv[1:] temp_page_flag = "--show-progress-page" + # Check for flag in the argv list. This doesn't use argparse + # because it's the only argument that's meant for the boostrap script. + # All the other flags will be passed to and parsed by the installer. if temp_page_flag in flags: - # Serve the loading page until TLJH builds with open("/var/run/index.html", "w+") as f: f.write(html) favicon_url="https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico" @@ -214,9 +220,13 @@ def main(): try: loading_page_server = HTTPServer(("", 80), LoaderPageRequestHandler) p = multiprocessing.Process(target=serve_forever, args=(loading_page_server,)) + # Serves the loading page until TLJH builds p.start() + + # Remove the flag from the args list, since it was only relevant to this script. + flags.remove("--show-progress-page") + # Pass the server's pid as a flag to the istaller - flags.remove(temp_page_flag) pid_flag = "--progress-page-server-pid" flags.extend([pid_flag, str(p.pid)]) except OSError: diff --git a/tljh/installer.py b/tljh/installer.py index 0a21967..1ad6969 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -507,10 +507,9 @@ def main(): ensure_jupyterlab_extensions() # Stop the http server with the loading page before traefik starts - if args.temporary_page: + if args.progress_page_server_pid: try: - os.kill(int(args.temporary_page), signal.SIGINT) - # Remove the pid file and the temporary html page + os.kill(args.progress_page_server_pid, signal.SIGINT) except Exception as e: pass From e65749bad1629d4c5932e3c813ee3bdcca776eae Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Tue, 18 Aug 2020 23:56:37 +0300 Subject: [PATCH 10/21] Test progress page started --- integration-tests/test_bootstrap.py | 43 +++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index fa02b68..a48065a 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -1,10 +1,13 @@ """ Test running bootstrap script in different circumstances """ +import concurrent.futures +import requests import subprocess from textwrap import dedent +import time -def run_bootstrap(container_name, image): +def run_bootstrap(container_name, image, flags=None): # stop container if it is already running subprocess.run([ 'docker', 'rm', '-f', container_name @@ -12,7 +15,7 @@ def run_bootstrap(container_name, image): # Start a detached Ubuntu 16.04 container subprocess.check_call([ - 'docker', 'run', '--detach', '--name', container_name, image, + 'docker', 'run', '--detach', '--publish', '12000:80', '--name', container_name, image, '/bin/bash', '-c', 'sleep 1000s' ]) # Install python3 inside the ubuntu container @@ -20,9 +23,13 @@ def run_bootstrap(container_name, image): subprocess.check_output([ 'docker', 'exec', container_name, 'apt-get', 'update' ]) + if flags: + pkgs = ['python3', 'systemd', 'git'] + else: + pkgs = ['python3'] subprocess.check_output([ - 'docker', 'exec', container_name, 'apt-get', 'install', '--yes', 'python3' - ]) + 'docker', 'exec', container_name, 'apt-get', 'install', '--yes'] + pkgs + ) # Copy only the bootstrap script to container, so this is faster subprocess.check_call([ 'docker', @@ -34,7 +41,7 @@ def run_bootstrap(container_name, image): return subprocess.run([ 'docker', 'exec', '-i', container_name, 'python3', '/srv/bootstrap/bootstrap.py' - ], check=False, stdout=subprocess.PIPE, encoding='utf-8') + ] + flags, check=False, stdout=subprocess.PIPE, encoding='utf-8') def test_ubuntu_too_old(): """ @@ -54,3 +61,29 @@ def test_inside_no_systemd_docker(): For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html """).strip() assert output.returncode == 1 + + +def verify_progress_page(expected_status_code): + progress_page_status = False + while True: + try: + r = requests.get('http://127.0.0.1:12000/index.html') + status = r.status_code + if status == expected_status_code: + progress_page_status = True + break; + except Exception as e: + time.sleep(5) + continue; + + return progress_page_status + +def test_progress_page(): + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(run_bootstrap, 'progress-page', 'ubuntu:18.04', ['--show-progress-page']) + + # Check if progress page started + started = verify_progress_page(200) + assert started + + return_value = future.result() From 15c5733a57ad263449ca8bd7644f258c34ac8ef2 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 19 Aug 2020 02:33:17 +0300 Subject: [PATCH 11/21] Test progress page stopped --- integration-tests/test_bootstrap.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index a48065a..e818633 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -23,6 +23,7 @@ def run_bootstrap(container_name, image, flags=None): subprocess.check_output([ 'docker', 'exec', container_name, 'apt-get', 'update' ]) + if flags: pkgs = ['python3', 'systemd', 'git'] else: @@ -63,27 +64,35 @@ def test_inside_no_systemd_docker(): assert output.returncode == 1 -def verify_progress_page(expected_status_code): +def verify_progress_page(expected_status_code, timeout): progress_page_status = False - while True: + start = time.time() + print("in verify_progress_page") + while not progress_page_status and (time.time() - start < timeout): try: - r = requests.get('http://127.0.0.1:12000/index.html') - status = r.status_code - if status == expected_status_code: + resp = requests.get('http://127.0.0.1:12000/index.html') + if resp.status_code == expected_status_code: progress_page_status = True break; except Exception as e: - time.sleep(5) + time.sleep(2) continue; return progress_page_status def test_progress_page(): with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(run_bootstrap, 'progress-page', 'ubuntu:18.04', ['--show-progress-page']) + installer = executor.submit(run_bootstrap, 'progress-page', 'ubuntu:18.04', ['--show-progress-page']) + print("started installer") # Check if progress page started - started = verify_progress_page(200) + started = verify_progress_page(expected_status_code=200, timeout=120) + print("started") assert started - return_value = future.result() + return_value = installer.result() + print("return value") + + # Check if progress page stopped + stopped = verify_progress_page(expected_status_code=404, timeout=120) + assert stopped From 05e9659763a3027bd42f936d729b1ae7fcad3844 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 19 Aug 2020 15:02:03 +0300 Subject: [PATCH 12/21] Remove debug statements --- integration-tests/test_bootstrap.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index e818633..58fa065 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -67,7 +67,6 @@ def test_inside_no_systemd_docker(): def verify_progress_page(expected_status_code, timeout): progress_page_status = False start = time.time() - print("in verify_progress_page") while not progress_page_status and (time.time() - start < timeout): try: resp = requests.get('http://127.0.0.1:12000/index.html') @@ -83,15 +82,12 @@ def verify_progress_page(expected_status_code, timeout): def test_progress_page(): with concurrent.futures.ThreadPoolExecutor() as executor: installer = executor.submit(run_bootstrap, 'progress-page', 'ubuntu:18.04', ['--show-progress-page']) - print("started installer") # Check if progress page started started = verify_progress_page(expected_status_code=200, timeout=120) - print("started") assert started return_value = installer.result() - print("return value") # Check if progress page stopped stopped = verify_progress_page(expected_status_code=404, timeout=120) From 884f3decddc267127888bf56cca7801b103d794e Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Fri, 21 Aug 2020 18:03:21 +0300 Subject: [PATCH 13/21] Log info about progress page server status --- tljh/installer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tljh/installer.py b/tljh/installer.py index 1ad6969..3712416 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -506,11 +506,14 @@ def main(): ensure_jupyterhub_package(HUB_ENV_PREFIX) ensure_jupyterlab_extensions() - # Stop the http server with the loading page before traefik starts + # Stop the http server with the progress page before traefik starts if args.progress_page_server_pid: try: os.kill(args.progress_page_server_pid, signal.SIGINT) + # Log and print the message to make testing easier + print("Progress page server stopped successfully.") except Exception as e: + logger.error(f"Couldn't stop the progress page server. Exception was {e}.") pass ensure_jupyterhub_service(HUB_ENV_PREFIX) From 62bd60e23c6f6bb2a7fff833d55e33041d3e8bdc Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Fri, 21 Aug 2020 18:03:57 +0300 Subject: [PATCH 14/21] Update boostrap tests --- integration-tests/test_bootstrap.py | 134 +++++++++++++++++++--------- 1 file changed, 92 insertions(+), 42 deletions(-) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 58fa065..79565b2 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -2,65 +2,111 @@ Test running bootstrap script in different circumstances """ import concurrent.futures +import os import requests import subprocess from textwrap import dedent import time -def run_bootstrap(container_name, image, flags=None): - # stop container if it is already running - subprocess.run([ - 'docker', 'rm', '-f', container_name - ]) - - # Start a detached Ubuntu 16.04 container - subprocess.check_call([ - 'docker', 'run', '--detach', '--publish', '12000:80', '--name', container_name, image, - '/bin/bash', '-c', 'sleep 1000s' - ]) + +def start_container(container_name, image, show_progress_page): + run_flags = [ + "--detach", + "--name", + container_name, + image, + "/bin/bash", + "-c", + "sleep 1000s", + ] + if show_progress_page: + # Use port-forwarding to be able to access the progress page when it starts + run_flags = ["--publish", "12000:80"] + run_flags + + # Start a detached container + subprocess.check_call(["docker", "run"] + run_flags) + + +def install_pkgs(container_name, show_progress_page): # Install python3 inside the ubuntu container # There is no trusted Ubuntu+Python3 container we can use - subprocess.check_output([ - 'docker', 'exec', container_name, 'apt-get', 'update' - ]) + pkgs = ["python3"] + if show_progress_page: + pkgs += ["systemd", "git"] + # Create the sudoers dir, so that the installer succesfully gets to the + # point of starting jupyterhub and stopping the progress page server. + subprocess.check_output( + ["docker", "exec", container_name, "mkdir", "-p", "etc/sudoers.d"] + ) - if flags: - pkgs = ['python3', 'systemd', 'git'] - else: - pkgs = ['python3'] - subprocess.check_output([ - 'docker', 'exec', container_name, 'apt-get', 'install', '--yes'] + pkgs + subprocess.check_output(["docker", "exec", container_name, "apt-get", "update"]) + subprocess.check_output( + ["docker", "exec", container_name, "apt-get", "install", "--yes"] + pkgs ) - # Copy only the bootstrap script to container, so this is faster - subprocess.check_call([ - 'docker', - 'cp', - 'bootstrap/', f'{container_name}:/srv' - ]) + + +def get_bootstrap_script_location(container_name, show_progress_page): + # Copy only the bootstrap script to container when progress page not enabled, to be faster + source_path = "bootstrap/" + bootstrap_script = "/srv/src/bootstrap.py" + if show_progress_page: + source_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir) + ) + bootstrap_script = "/srv/src/bootstrap/bootstrap.py" + + subprocess.check_call(["docker", "cp", source_path, f"{container_name}:/srv/src"]) + return bootstrap_script + + +def run_bootstrap(container_name, image, show_progress_page=False): + # stop container if it is already running + subprocess.run(["docker", "rm", "-f", container_name]) + + start_container(container_name, image, show_progress_page) + install_pkgs(container_name, show_progress_page) + + bootstrap_script = get_bootstrap_script_location(container_name, show_progress_page) + + exec_flags = ["-i", container_name, "python3", bootstrap_script] + if show_progress_page: + exec_flags = ( + ["-e", "TLJH_BOOTSTRAP_DEV=yes, TLJH_BOOTSTRAP_PIP_SPEC=/srv/src"] + + exec_flags + + ["--show-progress-page"] + ) # Run bootstrap script, return the output - return subprocess.run([ - 'docker', 'exec', '-i', container_name, - 'python3', '/srv/bootstrap/bootstrap.py' - ] + flags, check=False, stdout=subprocess.PIPE, encoding='utf-8') + return subprocess.run( + ["docker", "exec"] + exec_flags, + check=False, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + def test_ubuntu_too_old(): """ Error with a useful message when running in older Ubuntu """ - output = run_bootstrap('old-distro-test', 'ubuntu:16.04') - assert output.stdout == 'The Littlest JupyterHub requires Ubuntu 18.04 or higher\n' + output = run_bootstrap("old-distro-test", "ubuntu:16.04") + assert output.stdout == "The Littlest JupyterHub requires Ubuntu 18.04 or higher\n" assert output.returncode == 1 def test_inside_no_systemd_docker(): - output = run_bootstrap('plain-docker-test', 'ubuntu:18.04') - assert output.stdout.strip() == dedent(""" + output = run_bootstrap("plain-docker-test", "ubuntu:18.04") + assert ( + output.stdout.strip() + == dedent( + """ Systemd is required to run TLJH Running inside a docker container without systemd isn't supported We recommend against running a production TLJH instance inside a docker container For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html - """).strip() + """ + ).strip() + ) assert output.returncode == 1 @@ -69,26 +115,30 @@ def verify_progress_page(expected_status_code, timeout): start = time.time() while not progress_page_status and (time.time() - start < timeout): try: - resp = requests.get('http://127.0.0.1:12000/index.html') + resp = requests.get("http://127.0.0.1:12000/index.html") if resp.status_code == expected_status_code: progress_page_status = True - break; + break except Exception as e: time.sleep(2) - continue; + continue return progress_page_status + def test_progress_page(): with concurrent.futures.ThreadPoolExecutor() as executor: - installer = executor.submit(run_bootstrap, 'progress-page', 'ubuntu:18.04', ['--show-progress-page']) + installer = executor.submit( + run_bootstrap, "progress-page", "ubuntu:18.04", True + ) # Check if progress page started started = verify_progress_page(expected_status_code=200, timeout=120) assert started - return_value = installer.result() + # This will fail start tljh but should successfully get to the point + # Where it stops the progress page server. + output = installer.result() # Check if progress page stopped - stopped = verify_progress_page(expected_status_code=404, timeout=120) - assert stopped + assert "Progress page server stopped successfully." in output.stdout From a9fe4e771a6ff811633595b2be6a568f446f0a46 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Fri, 21 Aug 2020 23:47:23 +0300 Subject: [PATCH 15/21] Install requirement --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6a3d46..8ec75b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,6 +110,7 @@ commands: - run: name: Run bootstrap checks command: | + pip install requests py.test integration-tests/test_bootstrap.py From ebabe682c3fdc6cc33782f21412d2db734773f12 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sat, 22 Aug 2020 00:13:16 +0300 Subject: [PATCH 16/21] Reorder tests --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ec75b9..3a092da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,7 +110,7 @@ commands: - run: name: Run bootstrap checks command: | - pip install requests + python3 -m pip install requests py.test integration-tests/test_bootstrap.py @@ -170,15 +170,14 @@ jobs: - build_systemd_image + - bootstrap_checks + - basic_tests - admin_tests - plugin_tests - - bootstrap_checks - - upgrade-test: docker: - image: docker:18.05.0-ce-git From baf2c35f1f1b1980dcc3a95c04bfff66fb33f43b Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sat, 22 Aug 2020 15:57:40 +0300 Subject: [PATCH 17/21] Show tests output --- .circleci/config.yml | 2 +- integration-tests/test_bootstrap.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a092da..734ae7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,7 +111,7 @@ commands: name: Run bootstrap checks command: | python3 -m pip install requests - py.test integration-tests/test_bootstrap.py + py.test integration-tests/test_bootstrap.py -s jobs: diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 79565b2..cc6547b 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -71,7 +71,7 @@ def run_bootstrap(container_name, image, show_progress_page=False): exec_flags = ["-i", container_name, "python3", bootstrap_script] if show_progress_page: exec_flags = ( - ["-e", "TLJH_BOOTSTRAP_DEV=yes, TLJH_BOOTSTRAP_PIP_SPEC=/srv/src"] + ["-e", "TLJH_BOOTSTRAP_DEV=yes", "-e", "TLJH_BOOTSTRAP_PIP_SPEC=/srv/src"] + exec_flags + ["--show-progress-page"] ) From 025063b18924d4f6e7d4ef773f841a02a79052d1 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Sat, 22 Aug 2020 16:11:24 +0300 Subject: [PATCH 18/21] Log request error --- integration-tests/test_bootstrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index cc6547b..8a16f60 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -120,6 +120,7 @@ def verify_progress_page(expected_status_code, timeout): progress_page_status = True break except Exception as e: + print(e) time.sleep(2) continue From 66c74849e64f15f7da6196e79a377ba4a0662af4 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 24 Aug 2020 13:01:54 +0300 Subject: [PATCH 19/21] Check progress page up with curl --- .circleci/config.yml | 1 - integration-tests/test_bootstrap.py | 51 +++++++++++++++-------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 734ae7f..2aa989d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,7 +110,6 @@ commands: - run: name: Run bootstrap checks command: | - python3 -m pip install requests py.test integration-tests/test_bootstrap.py -s diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 8a16f60..4d3aea5 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -3,36 +3,17 @@ Test running bootstrap script in different circumstances """ import concurrent.futures import os -import requests import subprocess from textwrap import dedent import time -def start_container(container_name, image, show_progress_page): - run_flags = [ - "--detach", - "--name", - container_name, - image, - "/bin/bash", - "-c", - "sleep 1000s", - ] - if show_progress_page: - # Use port-forwarding to be able to access the progress page when it starts - run_flags = ["--publish", "12000:80"] + run_flags - - # Start a detached container - subprocess.check_call(["docker", "run"] + run_flags) - - def install_pkgs(container_name, show_progress_page): # Install python3 inside the ubuntu container # There is no trusted Ubuntu+Python3 container we can use pkgs = ["python3"] if show_progress_page: - pkgs += ["systemd", "git"] + pkgs += ["systemd", "git", "curl"] # Create the sudoers dir, so that the installer succesfully gets to the # point of starting jupyterhub and stopping the progress page server. subprocess.check_output( @@ -63,7 +44,21 @@ def run_bootstrap(container_name, image, show_progress_page=False): # stop container if it is already running subprocess.run(["docker", "rm", "-f", container_name]) - start_container(container_name, image, show_progress_page) + # Start a detached container + subprocess.check_call( + [ + "docker", + "run", + "--detach", + "--name", + container_name, + image, + "/bin/bash", + "-c", + "sleep 1000s", + ] + ) + install_pkgs(container_name, show_progress_page) bootstrap_script = get_bootstrap_script_location(container_name, show_progress_page) @@ -115,12 +110,20 @@ def verify_progress_page(expected_status_code, timeout): start = time.time() while not progress_page_status and (time.time() - start < timeout): try: - resp = requests.get("http://127.0.0.1:12000/index.html") - if resp.status_code == expected_status_code: + resp = subprocess.check_output( + [ + "docker", + "exec", + "progress-page", + "curl", + "-i", + "http://localhost/index.html", + ] + ) + if b"HTTP/1.0 200 OK" in resp: progress_page_status = True break except Exception as e: - print(e) time.sleep(2) continue From dcb6186561696eef1d2769e28ed5c9c1b6e1b8c2 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Mon, 24 Aug 2020 15:51:55 +0300 Subject: [PATCH 20/21] Document tljh progress page --- docs/topic/customizing-installer.rst | 29 ++++++++++++++++++++++++++++ docs/topic/installer-actions.rst | 18 +++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/docs/topic/customizing-installer.rst b/docs/topic/customizing-installer.rst index dd45d2b..713254c 100644 --- a/docs/topic/customizing-installer.rst +++ b/docs/topic/customizing-installer.rst @@ -17,6 +17,35 @@ This page documents the various options you can pass as commandline parameters t .. _topic/customizing-installer/admin: +Serving a temporary "TLJH is building" page +=========================================== +``--show-progress-page`` serves a temporary "TLJH is building" progress page while TLJH is building. + +.. image:: ../images/tljh-is-building-page.gif + :alt: Temporary progress page while TLJH is building + +* The page will be accessible at ``http:///index.html`` in your browser. + When TLJH installation is complete, the progress page page will stop and you will be able + to access TLJH as usually at ``http:///index.html``. +* From the progress page, you will also be able to access the installation logs, by clicking the + **Logs** button or by going directly to ``http:///logs`` in your browser. + To update the logs, refresh the page. + +.. note:: + + The ``http:///index.html`` page refreshes itself automatically every 30s. + When JupyterHub starts, a JupyterHub 404 HTTP error message (*Jupyter has lots of moons, but this is not one...*) + will be shown instead of the progress page. This means JupyterHub was started succesfully and you can access it + either by clicking the `Control Panel` button or by going to ``http:///`` directly. + +For example, to enable the progress page and add the first *admin* user, you would run: + +.. code-block:: bash + + curl -L https://tljh.jupyter.org/bootstrap.py \ + | sudo python3 - \ + --admin admin --showprogress-page + Adding admin users =================== diff --git a/docs/topic/installer-actions.rst b/docs/topic/installer-actions.rst index e5991a2..e655f66 100644 --- a/docs/topic/installer-actions.rst +++ b/docs/topic/installer-actions.rst @@ -139,6 +139,24 @@ to be used and modified solely by these services. sudo rm -rf /opt/tljh/state +Progress page files +=================== + +If you ran the TLJH installer with the `--show-progress-page` flag, then two files have been +added to your system to help serving the progress page: + +* ``/var/run/index.html`` - the main progress page +* ``/var/run/favicon.ico`` - the JupyterHub icon + +.. note:: + If you try to remove TLJH, revert this action using: + + .. code-block:: bash + + sudo rm /var/run/index.html + sudo rm /var/run/favicon.ico + + User groups =========== From a82871a789b4781d957efee13836be901ccd4b08 Mon Sep 17 00:00:00 2001 From: Yuvi Panda Date: Thu, 3 Sep 2020 12:50:05 +0530 Subject: [PATCH 21/21] Don't send people to /index.html after installation --- docs/topic/customizing-installer.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topic/customizing-installer.rst b/docs/topic/customizing-installer.rst index 713254c..b18febc 100644 --- a/docs/topic/customizing-installer.rst +++ b/docs/topic/customizing-installer.rst @@ -26,7 +26,7 @@ Serving a temporary "TLJH is building" page * The page will be accessible at ``http:///index.html`` in your browser. When TLJH installation is complete, the progress page page will stop and you will be able - to access TLJH as usually at ``http:///index.html``. + to access TLJH as usually at ``http://``. * From the progress page, you will also be able to access the installation logs, by clicking the **Logs** button or by going directly to ``http:///logs`` in your browser. To update the logs, refresh the page.