zhangyiss/the-littlest-jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/the-littlest-jupyterhub.git synced 2025-12-18 21:54:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor bootstrap.py script for readability

Browse Source
This commit is contained in:
Erik Sundell
2021-10-18 00:01:55 +02:00
parent d9e525d853
commit ba84229c8f
2 changed files with 138 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
bootstrap/bootstrap.py
View File

@@ -8,9 +8,34 @@ This script is run as:
curl <script-url> | sudo python3 - curl <script-url> | sudo python3 -
Constraints: Constraints:
- Entire script should be compatible with Python 3.6 (We run on Ubuntu 18.04+)
- Script should parse in Python 3.4 (since we exit with useful error message on Ubuntu 14.04+) - The entire script should be compatible with Python 3.6, which is the on
- Use stdlib modules only Ubuntu 18.04+.
- The script should parse in Python 3.5 as we print error messages on Ubuntu
16.04+ that comes with Python 3.5 by default. This means no f-strings.
- The script must depend only on stdlib modules, as no previous installation
of dependencies can be assumed.
Environment variables:
TLJH_INSTALL_PREFIX Defaults to "/opt/tljh", determines the location
of the tljh installations root folder.
TLJH_BOOTSTRAP_PIP_SPEC From this location, the bootstrap script will
pip install --upgrade the tljh installer.
TLJH_BOOTSTRAP_DEV Determines if --editable is passed when
installing the tljh installer. Pass the values
yes or no.
Command line flags:
The bootstrap.py script accept the following command line flags. All other
flags are passed through to the tljh installer without interception by this
script.
--show-progress-page Starts a local web server listening on port 80 where
logs can be accessed during installation. If this is
passed, it will pass --progress-page-server-pid=<pid>
to the tljh installer for later termination.
""" """
import os import os
from http.server import SimpleHTTPRequestHandler, HTTPServer from http.server import SimpleHTTPRequestHandler, HTTPServer
@@ -21,10 +46,11 @@ import logging
import shutil import shutil
import urllib.request import urllib.request
html = """ progress_page_favicon_url = "https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico"
progress_page_html = """
<html> <html>
<head> <head>
<title>The Littlest Jupyterhub</title> <title>The Littlest Jupyterhub</title>
</head> </head>
<body> <body>
<meta http-equiv="refresh" content="30" > <meta http-equiv="refresh" content="30" >
@@ -32,7 +58,7 @@ html = """
<meta name="viewport" content="width=device-width"> <meta name="viewport" content="width=device-width">
<img class="logo" src="https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/docs/images/logo/logo.png"> <img class="logo" src="https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/docs/images/logo/logo.png">
<div class="loader center"></div> <div class="loader center"></div>
<div class="center main-msg">Please wait while your TLJH is building...</div> <div class="center main-msg">Please wait while your TLJH is setting up...</div>
<div class="center logs-msg">Click the button below to see the logs</div> <div class="center logs-msg">Click the button below to see the logs</div>
<div class="center tip" >Tip: to update the logs, refresh the page</div> <div class="center tip" >Tip: to update the logs, refresh the page</div>
<button class="logs-button center" onclick="window.location.href='/logs'">View logs</button> <button class="logs-button center" onclick="window.location.href='/logs'">View logs</button>
@@ -99,25 +125,14 @@ html = """
</style> </style>
</head> </head>
</html> </html>
""" """
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def get_os_release_variable(key):
"""
Return value for key from /etc/os-release
/etc/os-release is a bash file, so should use bash to parse it. # This function is needed both by the process starting this script, and by the
# TLJH installer that this script execs in the end. Make sure its replica at
Returns empty string if key is not found. # tljh/utils.py stays in sync with this version!
"""
return subprocess.check_output([
'/bin/bash', '-c',
"source /etc/os-release && echo ${{{key}}}".format(key=key)
]).decode().strip()
# Copied into tljh/utils.py. Make sure the copies are exactly the same!
def run_subprocess(cmd, *args, **kwargs): def run_subprocess(cmd, *args, **kwargs):
""" """
Run given cmd with smart output behavior. Run given cmd with smart output behavior.
@@ -147,11 +162,26 @@ def run_subprocess(cmd, *args, **kwargs):
# For now, prioritizing human readability over machine readability. # For now, prioritizing human readability over machine readability.
logger.debug(proc.stdout.decode()) logger.debug(proc.stdout.decode())
def validate_host():
def ensure_host_system_can_install_tljh():
""" """
Make sure TLJH is installable in current host Check if TLJH is installable in current host system and exit with a clear
error message otherwise.
""" """
# Support only Ubuntu 18.04+ def get_os_release_variable(key):
"""
Return value for key from /etc/os-release
/etc/os-release is a bash file, so should use bash to parse it.
Returns empty string if key is not found.
"""
return subprocess.check_output([
'/bin/bash', '-c',
"source /etc/os-release && echo ${{{key}}}".format(key=key)
]).decode().strip()
# Require Ubuntu 18.04+
distro = get_os_release_variable('ID') distro = get_os_release_variable('ID')
version = float(get_os_release_variable('VERSION_ID')) version = float(get_os_release_variable('VERSION_ID'))
if distro != 'ubuntu': if distro != 'ubuntu':
@@ -161,20 +191,18 @@ def validate_host():
print('The Littlest JupyterHub requires Ubuntu 18.04 or higher') print('The Littlest JupyterHub requires Ubuntu 18.04 or higher')
sys.exit(1) sys.exit(1)
if sys.version_info < (3, 5): # Require systemd (systemctl is a part of systemd)
print("bootstrap.py must be run with at least Python 3.5") if not shutil.which('systemd') or not shutil.which('systemctl'):
sys.exit(1)
if not (shutil.which('systemd') and shutil.which('systemctl')):
print("Systemd is required to run TLJH") print("Systemd is required to run TLJH")
# Only fail running inside docker if systemd isn't present # Provide additional information about running in docker containers
if os.path.exists('/.dockerenv'): if os.path.exists('/.dockerenv'):
print("Running inside a docker container without systemd isn't supported") print("Running inside a docker container without systemd isn't supported")
print("We recommend against running a production TLJH instance inside a docker container") print("We recommend against running a production TLJH instance inside a docker container")
print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html")
sys.exit(1) sys.exit(1)
class LoaderPageRequestHandler(SimpleHTTPRequestHandler):
class ProgressPageRequestHandler(SimpleHTTPRequestHandler):
def do_GET(self): def do_GET(self):
if self.path == "/logs": if self.path == "/logs":
with open("/opt/tljh/installer.log", "r") as log_file: with open("/opt/tljh/installer.log", "r") as log_file:
@@ -197,45 +225,60 @@ class LoaderPageRequestHandler(SimpleHTTPRequestHandler):
else: else:
SimpleHTTPRequestHandler.send_error(self, code=403) SimpleHTTPRequestHandler.send_error(self, code=403)
def serve_forever(server):
try:
server.serve_forever()
except KeyboardInterrupt:
pass
def main(): def main():
flags = sys.argv[1:] """
temp_page_flag = "--show-progress-page" This script intercepts the --show-progress-page flag, but all other flags
are passed through to the TLJH installer script.
# Check for flag in the argv list. This doesn't use argparse The --show-progress-page flag indicates that the bootstrap script should
# because it's the only argument that's meant for the boostrap script. start a local webserver temporarily and report its installation progress via
# All the other flags will be passed to and parsed by the installer. a web site served locally on port 80.
if temp_page_flag in flags: """
with open("/var/run/index.html", "w+") as f: ensure_host_system_can_install_tljh()
f.write(html)
favicon_url="https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/share/jupyterhub/static/favicon.ico"
urllib.request.urlretrieve(favicon_url, "/var/run/favicon.ico")
# If the bootstrap is run to upgrade TLJH, then this will raise an "Address already in use" error
try:
loading_page_server = HTTPServer(("", 80), LoaderPageRequestHandler)
p = multiprocessing.Process(target=serve_forever, args=(loading_page_server,))
# Serves the loading page until TLJH builds
p.start()
# Remove the flag from the args list, since it was only relevant to this script. # Various related constants
flags.remove("--show-progress-page")
# Pass the server's pid as a flag to the istaller
pid_flag = "--progress-page-server-pid"
flags.extend([pid_flag, str(p.pid)])
except OSError:
# Only serve the loading page when installing TLJH
pass
validate_host()
install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
hub_prefix = os.path.join(install_prefix, 'hub') hub_prefix = os.path.join(install_prefix, 'hub')
python_bin = os.path.join(hub_prefix, 'bin', 'python3')
pip_bin = os.path.join(hub_prefix, 'bin', 'pip')
initial_setup = not os.path.exists(python_bin)
# Attempt to start a web server to serve a progress page reporting
# installation progress.
tljh_installer_flags = sys.argv[1:]
if "--show-progress-page" in tljh_installer_flags:
# Remove the bootstrap specific flag and let all other flags pass
# through to the installer.
tljh_installer_flags.remove("--show-progress-page")
# Write HTML and a favicon to be served by our webserver
with open("/var/run/index.html", "w+") as f:
f.write(progress_page_html)
urllib.request.urlretrieve(progress_page_favicon_url, "/var/run/favicon.ico")
# If TLJH is already installed and Traefik is already running, port 80
# will be busy and we will get an "Address already in use" error. This
# is acceptable and we can ignore the error.
try:
# Serve the loading page until manually aborted or until the TLJH
# installer terminates the process
def serve_forever(server):
try:
server.serve_forever()
except KeyboardInterrupt:
pass
progress_page_server = HTTPServer(("", 80), ProgressPageRequestHandler)
p = multiprocessing.Process(target=serve_forever, args=(progress_page_server,))
p.start()
# Pass the server's pid to the installer for later termination
tljh_installer_flags.extend(["--progress-page-server-pid", str(p.pid)])
except OSError:
pass
# Set up logging to print to a file and to stderr # Set up logging to print to a file and to stderr
os.makedirs(install_prefix, exist_ok=True) os.makedirs(install_prefix, exist_ok=True)
@@ -252,15 +295,15 @@ def main():
stderr_logger.setFormatter(logging.Formatter('%(message)s')) stderr_logger.setFormatter(logging.Formatter('%(message)s'))
stderr_logger.setLevel(logging.INFO) stderr_logger.setLevel(logging.INFO)
logger.addHandler(stderr_logger) logger.addHandler(stderr_logger)
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
logger.info('Checking if TLJH is already installed...')
if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): if not initial_setup:
logger.info('TLJH already installed, upgrading...') logger.info('TLJH installer is already installed, upgrading...')
initial_setup = False
else: else:
logger.info('Setting up hub environment') logger.info("TLJH installer isn't installed, installing...")
initial_setup = True logger.info('Installing Python, venv, pip, and git via apt-get...')
# Install software-properties-common, so we can get add-apt-repository # Install software-properties-common, so we can get add-apt-repository
# That helps us make sure the universe repository is enabled, since # That helps us make sure the universe repository is enabled, since
# that's where the python3-pip package lives. In some very minimal base # that's where the python3-pip package lives. In some very minimal base
@@ -277,48 +320,38 @@ def main():
'python3-pip', 'python3-pip',
'git' 'git'
]) ])
logger.info('Installed python & virtual environment')
logger.info('Setting up virtual environment at {}'.format(hub_prefix))
os.makedirs(hub_prefix, exist_ok=True) os.makedirs(hub_prefix, exist_ok=True)
run_subprocess(['python3', '-m', 'venv', hub_prefix]) run_subprocess(['python3', '-m', 'venv', hub_prefix])
logger.info('Set up hub virtual environment')
if initial_setup:
logger.info('Setting up TLJH installer...')
else:
logger.info('Upgrading TLJH installer...')
pip_flags = ['--upgrade']
if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes':
pip_flags.append('--editable')
tljh_repo_path = os.environ.get(
'TLJH_BOOTSTRAP_PIP_SPEC',
'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git'
)
# Upgrade pip # Upgrade pip
run_subprocess([ logger.info('Upgrading pip...')
os.path.join(hub_prefix, 'bin', 'pip'), run_subprocess([pip_bin, 'install', '--upgrade', 'pip==20.0.*'])
'install',
'--upgrade',
'pip==20.0.*'
])
logger.info('Upgraded pip')
run_subprocess([
os.path.join(hub_prefix, 'bin', 'pip'),
'install'
] + pip_flags + [tljh_repo_path])
logger.info('Setup tljh package')
logger.info('Starting TLJH installer...') # Install/upgrade TLJH installer
os.execv( tljh_install_cmd = [pip_bin, 'install', '--upgrade']
os.path.join(hub_prefix, 'bin', 'python3'), if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes':
[ tljh_install_cmd.append('--editable')
os.path.join(hub_prefix, 'bin', 'python3'), tljh_install_cmd.append(
'-m', os.environ.get(
'tljh.installer', 'TLJH_BOOTSTRAP_PIP_SPEC',
] + flags 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git'
)
) )
if initial_setup:
logger.info('Installing TLJH installer...')
else:
logger.info('Upgrading TLJH installer...')
run_subprocess(tljh_install_cmd)
# Run TLJH installer
logger.info('Running TLJH installer...')
os.execv(python_bin, [python_bin, '-m', 'tljh.installer'] + tljh_installer_flags)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

3
tljh/utils.py
View File

@@ -10,6 +10,9 @@ import pluggy
from tljh import hooks from tljh import hooks
# This function is needed also by the bootstrap script that starts this
# installer script. Make sure its replica at bootstrap/bootstrap.py stays in
# sync with this version!
def run_subprocess(cmd, *args, **kwargs): def run_subprocess(cmd, *args, **kwargs):
""" """
Run given cmd with smart output behavior. Run given cmd with smart output behavior.