diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 5fcd7b1..de76307 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -8,9 +8,35 @@ This script is run as: curl | sudo python3 - Constraints: - - Entire script should be compatible with Python 3.6 (We run on Ubuntu 18.04+) - - Script should parse in Python 3.4 (since we exit with useful error message on Ubuntu 14.04+) - - Use stdlib modules only + + - The entire script should be compatible with Python 3.6, which is the on + Ubuntu 18.04+. + - The script should parse in Python 3.5 as we print error messages for using + Ubuntu 16.04+ which comes with Python 3.5 by default. This means no + f-strings can be used. + - The script must depend only on stdlib modules, as no previous installation + of dependencies can be assumed. + +Environment variables: + + TLJH_INSTALL_PREFIX Defaults to "/opt/tljh", determines the location + of the tljh installations root folder. + TLJH_BOOTSTRAP_PIP_SPEC From this location, the bootstrap script will + pip install --upgrade the tljh installer. + TLJH_BOOTSTRAP_DEV Determines if --editable is passed when + installing the tljh installer. Pass the values + yes or no. + +Command line flags: + + The bootstrap.py script accept the following command line flags. All other + flags are passed through to the tljh installer without interception by this + script. + + --show-progress-page Starts a local web server listening on port 80 where + logs can be accessed during installation. If this is + passed, it will pass --progress-page-server-pid= + to the tljh installer for later termination. """ import os from http.server import SimpleHTTPRequestHandler, HTTPServer @@ -21,10 +47,11 @@ import logging import shutil import urllib.request -html = """ +progress_page_favicon_url = "https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico" +progress_page_html = """ - The Littlest Jupyterhub + The Littlest Jupyterhub @@ -32,7 +59,7 @@ html = """
-
Please wait while your TLJH is building...
+
Please wait while your TLJH is setting up...
Click the button below to see the logs
Tip: to update the logs, refresh the page
@@ -99,25 +126,14 @@ html = """ - """ logger = logging.getLogger(__name__) -def get_os_release_variable(key): - """ - Return value for key from /etc/os-release - /etc/os-release is a bash file, so should use bash to parse it. - - Returns empty string if key is not found. - """ - return subprocess.check_output([ - '/bin/bash', '-c', - "source /etc/os-release && echo ${{{key}}}".format(key=key) - ]).decode().strip() - -# Copied into tljh/utils.py. Make sure the copies are exactly the same! +# This function is needed both by the process starting this script, and by the +# TLJH installer that this script execs in the end. Make sure its replica at +# tljh/utils.py stays in sync with this version! def run_subprocess(cmd, *args, **kwargs): """ Run given cmd with smart output behavior. @@ -147,11 +163,26 @@ def run_subprocess(cmd, *args, **kwargs): # For now, prioritizing human readability over machine readability. logger.debug(proc.stdout.decode()) -def validate_host(): + +def ensure_host_system_can_install_tljh(): """ - Make sure TLJH is installable in current host + Check if TLJH is installable in current host system and exit with a clear + error message otherwise. """ - # Support only Ubuntu 18.04+ + def get_os_release_variable(key): + """ + Return value for key from /etc/os-release + + /etc/os-release is a bash file, so should use bash to parse it. + + Returns empty string if key is not found. + """ + return subprocess.check_output([ + '/bin/bash', '-c', + "source /etc/os-release && echo ${{{key}}}".format(key=key) + ]).decode().strip() + + # Require Ubuntu 18.04+ distro = get_os_release_variable('ID') version = float(get_os_release_variable('VERSION_ID')) if distro != 'ubuntu': @@ -161,20 +192,23 @@ def validate_host(): print('The Littlest JupyterHub requires Ubuntu 18.04 or higher') sys.exit(1) - if sys.version_info < (3, 5): - print("bootstrap.py must be run with at least Python 3.5") + # Require Python 3.6+ + if sys.version_info < (3, 6): + print("bootstrap.py must be run with at least Python 3.6") sys.exit(1) - if not (shutil.which('systemd') and shutil.which('systemctl')): + # Require systemd (systemctl is a part of systemd) + if not shutil.which('systemd') or not shutil.which('systemctl'): print("Systemd is required to run TLJH") - # Only fail running inside docker if systemd isn't present + # Provide additional information about running in docker containers if os.path.exists('/.dockerenv'): print("Running inside a docker container without systemd isn't supported") print("We recommend against running a production TLJH instance inside a docker container") print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") sys.exit(1) -class LoaderPageRequestHandler(SimpleHTTPRequestHandler): + +class ProgressPageRequestHandler(SimpleHTTPRequestHandler): def do_GET(self): if self.path == "/logs": with open("/opt/tljh/installer.log", "r") as log_file: @@ -197,45 +231,60 @@ class LoaderPageRequestHandler(SimpleHTTPRequestHandler): else: SimpleHTTPRequestHandler.send_error(self, code=403) -def serve_forever(server): - try: - server.serve_forever() - except KeyboardInterrupt: - pass def main(): - flags = sys.argv[1:] - temp_page_flag = "--show-progress-page" + """ + This script intercepts the --show-progress-page flag, but all other flags + are passed through to the TLJH installer script. - # Check for flag in the argv list. This doesn't use argparse - # because it's the only argument that's meant for the boostrap script. - # All the other flags will be passed to and parsed by the installer. - if temp_page_flag in flags: - with open("/var/run/index.html", "w+") as f: - f.write(html) - favicon_url="https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico" - urllib.request.urlretrieve(favicon_url, "/var/run/favicon.ico") + The --show-progress-page flag indicates that the bootstrap script should + start a local webserver temporarily and report its installation progress via + a web site served locally on port 80. + """ + ensure_host_system_can_install_tljh() - # If the bootstrap is run to upgrade TLJH, then this will raise an "Address already in use" error - try: - loading_page_server = HTTPServer(("", 80), LoaderPageRequestHandler) - p = multiprocessing.Process(target=serve_forever, args=(loading_page_server,)) - # Serves the loading page until TLJH builds - p.start() - # Remove the flag from the args list, since it was only relevant to this script. - flags.remove("--show-progress-page") - - # Pass the server's pid as a flag to the istaller - pid_flag = "--progress-page-server-pid" - flags.extend([pid_flag, str(p.pid)]) - except OSError: - # Only serve the loading page when installing TLJH - pass - - validate_host() + # Various related constants install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') + python_bin = os.path.join(hub_prefix, 'bin', 'python3') + pip_bin = os.path.join(hub_prefix, 'bin', 'pip') + initial_setup = not os.path.exists(python_bin) + + + # Attempt to start a web server to serve a progress page reporting + # installation progress. + tljh_installer_flags = sys.argv[1:] + if "--show-progress-page" in tljh_installer_flags: + # Remove the bootstrap specific flag and let all other flags pass + # through to the installer. + tljh_installer_flags.remove("--show-progress-page") + + # Write HTML and a favicon to be served by our webserver + with open("/var/run/index.html", "w+") as f: + f.write(progress_page_html) + urllib.request.urlretrieve(progress_page_favicon_url, "/var/run/favicon.ico") + + # If TLJH is already installed and Traefik is already running, port 80 + # will be busy and we will get an "Address already in use" error. This + # is acceptable and we can ignore the error. + try: + # Serve the loading page until manually aborted or until the TLJH + # installer terminates the process + def serve_forever(server): + try: + server.serve_forever() + except KeyboardInterrupt: + pass + progress_page_server = HTTPServer(("", 80), ProgressPageRequestHandler) + p = multiprocessing.Process(target=serve_forever, args=(progress_page_server,)) + p.start() + + # Pass the server's pid to the installer for later termination + tljh_installer_flags.extend(["--progress-page-server-pid", str(p.pid)]) + except OSError: + pass + # Set up logging to print to a file and to stderr os.makedirs(install_prefix, exist_ok=True) @@ -252,15 +301,16 @@ def main(): stderr_logger.setFormatter(logging.Formatter('%(message)s')) stderr_logger.setLevel(logging.INFO) logger.addHandler(stderr_logger) + logger.setLevel(logging.DEBUG) - logger.info('Checking if TLJH is already installed...') - if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): - logger.info('TLJH already installed, upgrading...') - initial_setup = False + + if not initial_setup: + logger.info('Existing TLJH installation detected, upgrading...') else: - logger.info('Setting up hub environment') - initial_setup = True + logger.info('Existing TLJH installation not detected, installing...') + logger.info('Setting up hub environment...') + logger.info('Installing Python, venv, pip, and git via apt-get...') # Install software-properties-common, so we can get add-apt-repository # That helps us make sure the universe repository is enabled, since # that's where the python3-pip package lives. In some very minimal base @@ -277,48 +327,38 @@ def main(): 'python3-pip', 'git' ]) - logger.info('Installed python & virtual environment') + + logger.info('Setting up virtual environment at {}'.format(hub_prefix)) os.makedirs(hub_prefix, exist_ok=True) run_subprocess(['python3', '-m', 'venv', hub_prefix]) - logger.info('Set up hub virtual environment') - if initial_setup: - logger.info('Setting up TLJH installer...') - else: - logger.info('Upgrading TLJH installer...') - - pip_flags = ['--upgrade'] - if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes': - pip_flags.append('--editable') - tljh_repo_path = os.environ.get( - 'TLJH_BOOTSTRAP_PIP_SPEC', - 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git' - ) # Upgrade pip - run_subprocess([ - os.path.join(hub_prefix, 'bin', 'pip'), - 'install', - '--upgrade', - 'pip==20.0.*' - ]) - logger.info('Upgraded pip') + logger.info('Upgrading pip...') + run_subprocess([pip_bin, 'install', '--upgrade', 'pip==20.0.*']) - run_subprocess([ - os.path.join(hub_prefix, 'bin', 'pip'), - 'install' - ] + pip_flags + [tljh_repo_path]) - logger.info('Setup tljh package') - logger.info('Starting TLJH installer...') - os.execv( - os.path.join(hub_prefix, 'bin', 'python3'), - [ - os.path.join(hub_prefix, 'bin', 'python3'), - '-m', - 'tljh.installer', - ] + flags + # Install/upgrade TLJH installer + tljh_install_cmd = [pip_bin, 'install', '--upgrade'] + if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes': + tljh_install_cmd.append('--editable') + tljh_install_cmd.append( + os.environ.get( + 'TLJH_BOOTSTRAP_PIP_SPEC', + 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git' + ) ) + if initial_setup: + logger.info('Installing TLJH installer...') + else: + logger.info('Upgrading TLJH installer...') + run_subprocess(tljh_install_cmd) + + + # Run TLJH installer + logger.info('Running TLJH installer...') + os.execv(python_bin, [python_bin, '-m', 'tljh.installer'] + tljh_installer_flags) + if __name__ == '__main__': main() diff --git a/tljh/utils.py b/tljh/utils.py index 460b24d..2b4d085 100644 --- a/tljh/utils.py +++ b/tljh/utils.py @@ -10,6 +10,9 @@ import pluggy from tljh import hooks +# This function is needed also by the bootstrap script that starts this +# installer script. Make sure its replica at bootstrap/bootstrap.py stays in +# sync with this version! def run_subprocess(cmd, *args, **kwargs): """ Run given cmd with smart output behavior.