zhangyiss/the-littlest-jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/the-littlest-jupyterhub.git synced 2025-12-18 21:54:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #755 from consideRatio/pr/apply-black

Browse Source 
pre-commit: apply black formatting (and prettier on one yaml file)
This commit is contained in:
Erik Sundell
2021-11-04 11:35:09 +01:00
committed by GitHub
parent e75de14839 1120466b71
commit ad38fad96a
36 changed files with 1209 additions and 1092 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
.circleci/config.yml
View File

@@ -87,7 +87,6 @@ commands:
command: |
py.test integration-tests/test_bootstrap.py -s
jobs:
integration-test:
docker:
@@ -107,32 +106,32 @@ jobs:
- plugin_tests
upgrade-test:
docker:
- image: docker:18.05.0-ce-git
docker:
- image: docker:18.05.0-ce-git
steps:
- run:
name: Check upgrade testing
command: |
if [ "$CIRCLE_BRANCH" == "master" ]; then
echo "On master, no upgrade to test..."
circleci-agent step halt
else
echo "PR detected, testing upgrade..."
fi
- run:
name: setup python3
command: |
apk add --no-cache python3 pytest
- checkout
- setup_remote_docker
- build_systemd_image
- basic_tests:
upgrade: "--upgrade"
- admin_tests:
upgrade: "--upgrade"
- plugin_tests:
upgrade: "--upgrade"
steps:
- run:
name: Check upgrade testing
command: |
if [ "$CIRCLE_BRANCH" == "master" ]; then
echo "On master, no upgrade to test..."
circleci-agent step halt
else
echo "PR detected, testing upgrade..."
fi
- run:
name: setup python3
command: |
apk add --no-cache python3 pytest
- checkout
- setup_remote_docker
- build_systemd_image
- basic_tests:
upgrade: "--upgrade"
- admin_tests:
upgrade: "--upgrade"
- plugin_tests:
upgrade: "--upgrade"
workflows:
version: 2

164
.github/integration-test.py vendored
View File

@@ -10,7 +10,7 @@ def build_systemd_image(image_name, source_path, build_args=None):
Built image is tagged with image_name
"""
cmd = ['docker', 'build', '-t', image_name, source_path]
cmd = ["docker", "build", f"-t={image_name}", source_path]
if build_args:
cmd.extend([f"--build-arg={ba}" for ba in build_args])
subprocess.check_call(cmd)
@@ -25,19 +25,20 @@ def run_systemd_image(image_name, container_name, bootstrap_pip_spec):
Container named container_name will be started.
"""
cmd = [
'docker', 'run',
'--privileged',
'--mount', 'type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup',
'--detach',
'--name', container_name,
"docker",
"run",
"--privileged",
"--mount=type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup",
"--detach",
f"--name={container_name}",
# A bit less than 1GB to ensure TLJH runs on 1GB VMs.
# If this is changed all docs references to the required memory must be changed too.
'--memory', '900m',
"--memory=900m",
]
if bootstrap_pip_spec:
cmd.append('-e')
cmd.append(f'TLJH_BOOTSTRAP_PIP_SPEC={bootstrap_pip_spec}')
cmd.append("-e")
cmd.append(f"TLJH_BOOTSTRAP_PIP_SPEC={bootstrap_pip_spec}")
cmd.append(image_name)
@@ -49,83 +50,76 @@ def stop_container(container_name):
Stop & remove docker container if it exists.
"""
try:
subprocess.check_output([
'docker', 'inspect', container_name
], stderr=subprocess.STDOUT)
subprocess.check_output(
["docker", "inspect", container_name], stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError:
# No such container exists, nothing to do
return
subprocess.check_call([
'docker', 'rm', '-f', container_name
])
subprocess.check_call(["docker", "rm", "-f", container_name])
def run_container_command(container_name, cmd):
"""
Run cmd in a running container with a bash shell
"""
proc = subprocess.run([
'docker', 'exec',
'-t', container_name,
'/bin/bash', '-c', cmd
], check=True)
proc = subprocess.run(
["docker", "exec", "-t", container_name, "/bin/bash", "-c", cmd], check=True
)
def copy_to_container(container_name, src_path, dest_path):
"""
Copy files from src_path to dest_path inside container_name
"""
subprocess.check_call([
'docker', 'cp',
src_path, f'{container_name}:{dest_path}'
])
subprocess.check_call(["docker", "cp", src_path, f"{container_name}:{dest_path}"])
def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args):
def run_test(
image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args
):
"""
Wrapper that sets up tljh with installer_args & runs test_name
"""
stop_container(test_name)
run_systemd_image(image_name, test_name, bootstrap_pip_spec)
source_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir)
)
source_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
copy_to_container(test_name, os.path.join(source_path, 'bootstrap/.'), '/srv/src')
copy_to_container(test_name, os.path.join(source_path, 'integration-tests/'), '/srv/src')
copy_to_container(test_name, os.path.join(source_path, "bootstrap/."), "/srv/src")
copy_to_container(
test_name, os.path.join(source_path, "integration-tests/"), "/srv/src"
)
# These logs can be very relevant to debug a container startup failure
print(f"--- Start of logs from the container: {test_name}")
print(subprocess.check_output(['docker', 'logs', test_name]).decode())
print(subprocess.check_output(["docker", "logs", test_name]).decode())
print(f"--- End of logs from the container: {test_name}")
# Install TLJH from the default branch first to test upgrades
if upgrade:
run_container_command(
test_name,
'curl -L https://tljh.jupyter.org/bootstrap.py | python3 -'
test_name, "curl -L https://tljh.jupyter.org/bootstrap.py | python3 -"
)
run_container_command(
test_name,
f'python3 /srv/src/bootstrap.py {installer_args}'
)
run_container_command(test_name, f"python3 /srv/src/bootstrap.py {installer_args}")
# Install pkgs from requirements in hub's pip, where
# the bootstrap script installed the others
run_container_command(
test_name,
'/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt'
"/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt",
)
run_container_command(
test_name,
# We abort pytest after two failures as a compromise between wanting to
# avoid a flood of logs while still understanding if multiple tests
# would fail.
'/opt/tljh/hub/bin/python3 -m pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no {}'.format(
' '.join([os.path.join('/srv/src/integration-tests/', f) for f in test_files])
)
"/opt/tljh/hub/bin/python3 -m pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no {}".format(
" ".join(
[os.path.join("/srv/src/integration-tests/", f) for f in test_files]
)
),
)
@@ -133,72 +127,74 @@ def show_logs(container_name):
"""
Print logs from inside container to stdout
"""
run_container_command(container_name, "journalctl --no-pager")
run_container_command(
container_name,
'journalctl --no-pager'
)
run_container_command(
container_name,
'systemctl --no-pager status jupyterhub traefik'
container_name, "systemctl --no-pager status jupyterhub traefik"
)
def main():
argparser = argparse.ArgumentParser()
subparsers = argparser.add_subparsers(dest='action')
subparsers = argparser.add_subparsers(dest="action")
build_image_parser = subparsers.add_parser('build-image')
build_image_parser = subparsers.add_parser("build-image")
build_image_parser.add_argument(
"--build-arg",
action="append",
dest="build_args",
)
subparsers.add_parser('stop-container').add_argument(
'container_name'
subparsers.add_parser("stop-container").add_argument("container_name")
subparsers.add_parser("start-container").add_argument("container_name")
run_parser = subparsers.add_parser("run")
run_parser.add_argument("container_name")
run_parser.add_argument("command")
copy_parser = subparsers.add_parser("copy")
copy_parser.add_argument("container_name")
copy_parser.add_argument("src")
copy_parser.add_argument("dest")
run_test_parser = subparsers.add_parser("run-test")
run_test_parser.add_argument("--installer-args", default="")
run_test_parser.add_argument("--upgrade", action="store_true")
run_test_parser.add_argument(
"--bootstrap-pip-spec", nargs="?", default="", type=str
)
run_test_parser.add_argument("test_name")
run_test_parser.add_argument("test_files", nargs="+")
subparsers.add_parser('start-container').add_argument(
'container_name'
)
run_parser = subparsers.add_parser('run')
run_parser.add_argument('container_name')
run_parser.add_argument('command')
copy_parser = subparsers.add_parser('copy')
copy_parser.add_argument('container_name')
copy_parser.add_argument('src')
copy_parser.add_argument('dest')
run_test_parser = subparsers.add_parser('run-test')
run_test_parser.add_argument('--installer-args', default='')
run_test_parser.add_argument('--upgrade', action='store_true')
run_test_parser.add_argument('--bootstrap-pip-spec', nargs='?', default="", type=str)
run_test_parser.add_argument('test_name')
run_test_parser.add_argument('test_files', nargs='+')
show_logs_parser = subparsers.add_parser('show-logs')
show_logs_parser.add_argument('container_name')
show_logs_parser = subparsers.add_parser("show-logs")
show_logs_parser.add_argument("container_name")
args = argparser.parse_args()
image_name = 'tljh-systemd'
image_name = "tljh-systemd"
if args.action == 'run-test':
run_test(image_name, args.test_name, args.bootstrap_pip_spec, args.test_files, args.upgrade, args.installer_args)
elif args.action == 'show-logs':
if args.action == "run-test":
run_test(
image_name,
args.test_name,
args.bootstrap_pip_spec,
args.test_files,
args.upgrade,
args.installer_args,
)
elif args.action == "show-logs":
show_logs(args.container_name)
elif args.action == 'run':
elif args.action == "run":
run_container_command(args.container_name, args.command)
elif args.action == 'copy':
elif args.action == "copy":
copy_to_container(args.container_name, args.src, args.dest)
elif args.action == 'start-container':
elif args.action == "start-container":
run_systemd_image(image_name, args.container_name, args.bootstrap_pip_spec)
elif args.action == 'stop-container':
elif args.action == "stop-container":
stop_container(args.container_name)
elif args.action == 'build-image':
build_systemd_image(image_name, 'integration-tests', args.build_args)
elif args.action == "build-image":
build_systemd_image(image_name, "integration-tests", args.build_args)
if __name__ == '__main__':
if __name__ == "__main__":
main()

154
bootstrap/bootstrap.py
View File

@@ -144,21 +144,27 @@ def run_subprocess(cmd, *args, **kwargs):
In TLJH, this sends successful output to the installer log,
and failed output directly to the user's screen
"""
logger = logging.getLogger('tljh')
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs)
printable_command = ' '.join(cmd)
logger = logging.getLogger("tljh")
proc = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs
)
printable_command = " ".join(cmd)
if proc.returncode != 0:
# Our process failed! Show output to the user
logger.error('Ran {command} with exit code {code}'.format(
command=printable_command, code=proc.returncode
))
logger.error(
"Ran {command} with exit code {code}".format(
command=printable_command, code=proc.returncode
)
)
logger.error(proc.stdout.decode())
raise subprocess.CalledProcessError(cmd=cmd, returncode=proc.returncode)
else:
# This goes into installer.log
logger.debug('Ran {command} with exit code {code}'.format(
command=printable_command, code=proc.returncode
))
logger.debug(
"Ran {command} with exit code {code}".format(
command=printable_command, code=proc.returncode
)
)
# This produces multi line log output, unfortunately. Not sure how to fix.
# For now, prioritizing human readability over machine readability.
logger.debug(proc.stdout.decode())
@@ -169,6 +175,7 @@ def ensure_host_system_can_install_tljh():
Check if TLJH is installable in current host system and exit with a clear
error message otherwise.
"""
def get_os_release_variable(key):
"""
Return value for key from /etc/os-release
@@ -177,19 +184,26 @@ def ensure_host_system_can_install_tljh():
Returns empty string if key is not found.
"""
return subprocess.check_output([
'/bin/bash', '-c',
"source /etc/os-release && echo ${{{key}}}".format(key=key)
]).decode().strip()
return (
subprocess.check_output(
[
"/bin/bash",
"-c",
"source /etc/os-release && echo ${{{key}}}".format(key=key),
]
)
.decode()
.strip()
)
# Require Ubuntu 18.04+
distro = get_os_release_variable('ID')
version = float(get_os_release_variable('VERSION_ID'))
if distro != 'ubuntu':
print('The Littlest JupyterHub currently supports Ubuntu Linux only')
distro = get_os_release_variable("ID")
version = float(get_os_release_variable("VERSION_ID"))
if distro != "ubuntu":
print("The Littlest JupyterHub currently supports Ubuntu Linux only")
sys.exit(1)
elif float(version) < 18.04:
print('The Littlest JupyterHub requires Ubuntu 18.04 or higher')
print("The Littlest JupyterHub requires Ubuntu 18.04 or higher")
sys.exit(1)
# Require Python 3.6+
@@ -198,13 +212,17 @@ def ensure_host_system_can_install_tljh():
sys.exit(1)
# Require systemd (systemctl is a part of systemd)
if not shutil.which('systemd') or not shutil.which('systemctl'):
if not shutil.which("systemd") or not shutil.which("systemctl"):
print("Systemd is required to run TLJH")
# Provide additional information about running in docker containers
if os.path.exists('/.dockerenv'):
if os.path.exists("/.dockerenv"):
print("Running inside a docker container without systemd isn't supported")
print("We recommend against running a production TLJH instance inside a docker container")
print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html")
print(
"We recommend against running a production TLJH instance inside a docker container"
)
print(
"For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html"
)
sys.exit(1)
@@ -215,9 +233,9 @@ class ProgressPageRequestHandler(SimpleHTTPRequestHandler):
logs = log_file.read()
self.send_response(200)
self.send_header('Content-Type', 'text/plain; charset=utf-8')
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(logs.encode('utf-8'))
self.wfile.write(logs.encode("utf-8"))
elif self.path == "/index.html":
self.path = "/var/run/index.html"
return SimpleHTTPRequestHandler.do_GET(self)
@@ -226,7 +244,7 @@ class ProgressPageRequestHandler(SimpleHTTPRequestHandler):
return SimpleHTTPRequestHandler.do_GET(self)
elif self.path == "/":
self.send_response(302)
self.send_header('Location','/index.html')
self.send_header("Location", "/index.html")
self.end_headers()
else:
SimpleHTTPRequestHandler.send_error(self, code=403)
@@ -243,15 +261,13 @@ def main():
"""
ensure_host_system_can_install_tljh()
# Various related constants
install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
hub_prefix = os.path.join(install_prefix, 'hub')
python_bin = os.path.join(hub_prefix, 'bin', 'python3')
pip_bin = os.path.join(hub_prefix, 'bin', 'pip')
install_prefix = os.environ.get("TLJH_INSTALL_PREFIX", "/opt/tljh")
hub_prefix = os.path.join(install_prefix, "hub")
python_bin = os.path.join(hub_prefix, "bin", "python3")
pip_bin = os.path.join(hub_prefix, "bin", "pip")
initial_setup = not os.path.exists(python_bin)
# Attempt to start a web server to serve a progress page reporting
# installation progress.
tljh_installer_flags = sys.argv[1:]
@@ -276,8 +292,11 @@ def main():
server.serve_forever()
except KeyboardInterrupt:
pass
progress_page_server = HTTPServer(("", 80), ProgressPageRequestHandler)
p = multiprocessing.Process(target=serve_forever, args=(progress_page_server,))
p = multiprocessing.Process(
target=serve_forever, args=(progress_page_server,)
)
p.start()
# Pass the server's pid to the installer for later termination
@@ -285,32 +304,30 @@ def main():
except OSError:
pass
# Set up logging to print to a file and to stderr
os.makedirs(install_prefix, exist_ok=True)
file_logger_path = os.path.join(install_prefix, 'installer.log')
file_logger_path = os.path.join(install_prefix, "installer.log")
file_logger = logging.FileHandler(file_logger_path)
# installer.log should be readable only by root
os.chmod(file_logger_path, 0o500)
file_logger.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
file_logger.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
file_logger.setLevel(logging.DEBUG)
logger.addHandler(file_logger)
stderr_logger = logging.StreamHandler()
stderr_logger.setFormatter(logging.Formatter('%(message)s'))
stderr_logger.setFormatter(logging.Formatter("%(message)s"))
stderr_logger.setLevel(logging.INFO)
logger.addHandler(stderr_logger)
logger.setLevel(logging.DEBUG)
if not initial_setup:
logger.info('Existing TLJH installation detected, upgrading...')
logger.info("Existing TLJH installation detected, upgrading...")
else:
logger.info('Existing TLJH installation not detected, installing...')
logger.info('Setting up hub environment...')
logger.info('Installing Python, venv, pip, and git via apt-get...')
logger.info("Existing TLJH installation not detected, installing...")
logger.info("Setting up hub environment...")
logger.info("Installing Python, venv, pip, and git via apt-get...")
# In some very minimal base VM images, it looks like the "universe" apt
# package repository is disabled by default, causing bootstrapping to
@@ -323,45 +340,56 @@ def main():
#
apt_get_adjusted_env = os.environ.copy()
apt_get_adjusted_env["DEBIAN_FRONTEND"] = "noninteractive"
run_subprocess(['apt-get', 'update'])
run_subprocess(['apt-get', 'install', '--yes', 'software-properties-common'], env=apt_get_adjusted_env)
run_subprocess(['add-apt-repository', 'universe', '--yes'])
run_subprocess(['apt-get', 'update'])
run_subprocess(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'python3-pip', 'git'], env=apt_get_adjusted_env)
run_subprocess(["apt-get", "update"])
run_subprocess(
["apt-get", "install", "--yes", "software-properties-common"],
env=apt_get_adjusted_env,
)
run_subprocess(["add-apt-repository", "universe", "--yes"])
run_subprocess(["apt-get", "update"])
run_subprocess(
[
"apt-get",
"install",
"--yes",
"python3",
"python3-venv",
"python3-pip",
"git",
],
env=apt_get_adjusted_env,
)
logger.info('Setting up virtual environment at {}'.format(hub_prefix))
logger.info("Setting up virtual environment at {}".format(hub_prefix))
os.makedirs(hub_prefix, exist_ok=True)
run_subprocess(['python3', '-m', 'venv', hub_prefix])
run_subprocess(["python3", "-m", "venv", hub_prefix])
# Upgrade pip
# Keep pip version pinning in sync with the one in unit-test.yml!
# See changelog at https://pip.pypa.io/en/latest/news/#changelog
logger.info('Upgrading pip...')
run_subprocess([pip_bin, 'install', '--upgrade', 'pip==21.3.*'])
logger.info("Upgrading pip...")
run_subprocess([pip_bin, "install", "--upgrade", "pip==21.3.*"])
# Install/upgrade TLJH installer
tljh_install_cmd = [pip_bin, 'install', '--upgrade']
if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes':
tljh_install_cmd.append('--editable')
tljh_install_cmd = [pip_bin, "install", "--upgrade"]
if os.environ.get("TLJH_BOOTSTRAP_DEV", "no") == "yes":
tljh_install_cmd.append("--editable")
tljh_install_cmd.append(
os.environ.get(
'TLJH_BOOTSTRAP_PIP_SPEC',
'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git'
"TLJH_BOOTSTRAP_PIP_SPEC",
"git+https://github.com/jupyterhub/the-littlest-jupyterhub.git",
)
)
if initial_setup:
logger.info('Installing TLJH installer...')
logger.info("Installing TLJH installer...")
else:
logger.info('Upgrading TLJH installer...')
logger.info("Upgrading TLJH installer...")
run_subprocess(tljh_install_cmd)
# Run TLJH installer
logger.info('Running TLJH installer...')
os.execv(python_bin, [python_bin, '-m', 'tljh.installer'] + tljh_installer_flags)
logger.info("Running TLJH installer...")
os.execv(python_bin, [python_bin, "-m", "tljh.installer"] + tljh_installer_flags)
if __name__ == '__main__':
if __name__ == "__main__":
main()

44
docs/conf.py
View File

@@ -1,20 +1,22 @@
import os
source_suffix = ['.rst']
source_suffix = [".rst"]
project = 'The Littlest JupyterHub'
copyright = '2018, JupyterHub Team'
author = 'JupyterHub Team'
project = "The Littlest JupyterHub"
copyright = "2018, JupyterHub Team"
author = "JupyterHub Team"
# The short X.Y version
version = ''
version = ""
# The full version, including alpha/beta/rc tags
release = 'v0.1'
release = "v0.1"
# Enable MathJax for Math
extensions = ['sphinx.ext.mathjax',
'sphinx.ext.intersphinx',
'sphinx_copybutton']
extensions = [
"sphinx.ext.mathjax",
"sphinx.ext.intersphinx",
"sphinx_copybutton",
]
# The root toctree document.
root_doc = master_doc = "index"
@@ -22,27 +24,31 @@ root_doc = master_doc = "index"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store',
'install/custom.rst']
exclude_patterns = [
"_build",
"Thumbs.db",
".DS_Store",
"install/custom.rst",
]
intersphinx_mapping = {
'sphinx': ('http://www.sphinx-doc.org/en/master/', None),
"sphinx": ("http://www.sphinx-doc.org/en/master/", None),
}
intersphinx_cache_limit = 90 # days
intersphinx_cache_limit = 90 # days
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
html_theme = 'pydata_sphinx_theme'
html_theme = "pydata_sphinx_theme"
html_logo = 'images/logo/logo.png'
html_favicon = 'images/logo/favicon.ico'
html_logo = "images/logo/logo.png"
html_favicon = "images/logo/favicon.ico"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# Do this only if _static exists, otherwise this will error
here = os.path.dirname(os.path.abspath(__file__))
if os.path.exists(os.path.join(here, '_static')):
html_static_path = ['_static']
if os.path.exists(os.path.join(here, "_static")):
html_static_path = ["_static"]

2
integration-tests/conftest.py
View File

@@ -5,13 +5,13 @@ import os
from pytest import fixture
@fixture
def preserve_config(request):
"""Fixture to save and restore config around tests"""
# Import TLJH only when needed. This lets us run tests in places
# where TLJH is not installed - particularly, the 'distro check' test.
from tljh.config import CONFIG_FILE, reload_component
if os.path.exists(CONFIG_FILE):
with open(CONFIG_FILE) as f:
save_config = f.read()

23
integration-tests/plugins/simplest/tljh_simplest.py
View File

@@ -7,48 +7,49 @@ from tljh.hooks import hookimpl
@hookimpl
def tljh_extra_user_conda_packages():
return [
'hypothesis',
"hypothesis",
]
@hookimpl
def tljh_extra_user_pip_packages():
return [
'django',
"django",
]
@hookimpl
def tljh_extra_hub_pip_packages():
return [
'there',
"there",
]
@hookimpl
def tljh_extra_apt_packages():
return [
'sl',
"sl",
]
@hookimpl
def tljh_config_post_install(config):
# Put an arbitrary marker we can test for
config['simplest_plugin'] = {
'present': True
}
config["simplest_plugin"] = {"present": True}
@hookimpl
def tljh_custom_jupyterhub_config(c):
c.JupyterHub.authenticator_class = 'tmpauthenticator.TmpAuthenticator'
c.JupyterHub.authenticator_class = "tmpauthenticator.TmpAuthenticator"
@hookimpl
def tljh_post_install():
with open('test_post_install', 'w') as f:
f.write('123456789')
with open("test_post_install", "w") as f:
f.write("123456789")
@hookimpl
def tljh_new_user_create(username):
with open('test_new_user_create', 'w') as f:
with open("test_new_user_create", "w") as f:
f.write(username)

5
integration-tests/test_admin_installer.py
View File

@@ -10,7 +10,7 @@ async def test_admin_login():
Test if the admin that was added during install can login with
the password provided.
"""
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = "admin"
password = "admin"
@@ -19,6 +19,7 @@ async def test_admin_login():
# If user is not logged in, this will raise an exception
await u.ensure_server_simulate()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"username, password",
@@ -32,7 +33,7 @@ async def test_unsuccessful_login(username, password):
"""
Ensure nobody but the admin that was added during install can login
"""
hub_url = 'http://localhost'
hub_url = "http://localhost"
async with User(username, hub_url, partial(login_dummy, password="")) as u:
user_logged_in = await u.login()

7
integration-tests/test_bootstrap.py
View File

@@ -38,6 +38,7 @@ def get_bootstrap_script_location(container_name, show_progress_page):
subprocess.check_call(["docker", "cp", source_path, f"{container_name}:/srv/src"])
return bootstrap_script
# FIXME: Refactor this function to easier to understand using the following
# parameters
#
@@ -55,7 +56,9 @@ def get_bootstrap_script_location(container_name, show_progress_page):
# running against the systemd container that cab be built by
# integration-test.py.
#
def run_bootstrap_after_preparing_container(container_name, image, show_progress_page=False):
def run_bootstrap_after_preparing_container(
container_name, image, show_progress_page=False
):
"""
1. Stops old container
2. Starts --detached container
@@ -163,7 +166,7 @@ def test_progress_page():
run_bootstrap_after_preparing_container,
"progress-page",
f"ubuntu:{os.getenv('UBUNTU_VERSION', '20.04')}",
True
True,
)
# Check if progress page started

36
integration-tests/test_extensions.py
View File

@@ -6,39 +6,43 @@ def test_serverextensions():
Validate serverextensions we want are installed
"""
# jupyter-serverextension writes to stdout and stderr weirdly
proc = subprocess.run([
'/opt/tljh/user/bin/jupyter-serverextension',
'list', '--sys-prefix'
], stderr=subprocess.PIPE)
proc = subprocess.run(
["/opt/tljh/user/bin/jupyter-serverextension", "list", "--sys-prefix"],
stderr=subprocess.PIPE,
)
extensions = [
'jupyterlab 3.',
'nbgitpuller 1.',
'nteract_on_jupyter 2.1.',
'jupyter_resource_usage'
"jupyterlab 3.",
"nbgitpuller 1.",
"nteract_on_jupyter 2.1.",
"jupyter_resource_usage",
]
for e in extensions:
assert e in proc.stderr.decode()
def test_nbextensions():
"""
Validate nbextensions we want are installed & enabled
"""
# jupyter-nbextension writes to stdout and stderr weirdly
proc = subprocess.run([
'/opt/tljh/user/bin/jupyter-nbextension',
'list', '--sys-prefix'
], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
proc = subprocess.run(
["/opt/tljh/user/bin/jupyter-nbextension", "list", "--sys-prefix"],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
extensions = [
'jupyter_resource_usage/main',
"jupyter_resource_usage/main",
# This is what ipywidgets nbextension is called
'jupyter-js-widgets/extension'
"jupyter-js-widgets/extension",
]
for e in extensions:
assert f'{e} \x1b[32m enabled \x1b[0m' in proc.stdout.decode()
assert f"{e} \x1b[32m enabled \x1b[0m" in proc.stdout.decode()
# Ensure we have 'OK' messages in our stdout, to make sure everything is importable
assert proc.stderr.decode() == ' - Validating: \x1b[32mOK\x1b[0m\n' * len(extensions)
assert proc.stderr.decode() == " - Validating: \x1b[32mOK\x1b[0m\n" * len(
extensions
)

352
integration-tests/test_hub.py
View File

@@ -15,11 +15,11 @@ from tljh.normalize import generate_system_username
# Use sudo to invoke it, since this is how users invoke it.
# This catches issues with PATH
TLJH_CONFIG_PATH = ['sudo', 'tljh-config']
TLJH_CONFIG_PATH = ["sudo", "tljh-config"]
def test_hub_up():
r = requests.get('http://127.0.0.1')
r = requests.get("http://127.0.0.1")
r.raise_for_status()
@@ -30,20 +30,32 @@ async def test_user_code_execute():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
await u.start_kernel()
await u.assert_code_output("5 * 4", "20", 5, 5)
# Assert that the user exists
assert pwd.getpwnam(f'jupyter-{username}') is not None
assert pwd.getpwnam(f"jupyter-{username}") is not None
@pytest.mark.asyncio
@@ -57,17 +69,48 @@ async def test_user_server_started_with_custom_base_url():
hub_url = f"http://localhost{base_url}"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "base_url", base_url
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
# unset base_url to avoid problems with other tests
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'unset', 'base_url')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "unset", "base_url"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
@pytest.mark.asyncio
@@ -77,23 +120,41 @@ async def test_user_admin_add():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "add-item", "users.admin", username
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
# Assert that the user exists
assert pwd.getpwnam(f'jupyter-{username}') is not None
assert pwd.getpwnam(f"jupyter-{username}") is not None
# Assert that the user has admin rights
assert f'jupyter-{username}' in grp.getgrnam(
'jupyterhub-admins').gr_mem
assert f"jupyter-{username}" in grp.getgrnam("jupyterhub-admins").gr_mem
# FIXME: Make this test pass
@@ -107,33 +168,62 @@ async def test_user_admin_remove():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "add-item", "users.admin", username
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
# Assert that the user exists
assert pwd.getpwnam(f'jupyter-{username}') is not None
assert pwd.getpwnam(f"jupyter-{username}") is not None
# Assert that the user has admin rights
assert f'jupyter-{username}' in grp.getgrnam(
'jupyterhub-admins').gr_mem
assert f"jupyter-{username}" in grp.getgrnam("jupyterhub-admins").gr_mem
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "remove-item", "users.admin", username
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
await u.stop_server()
await u.ensure_server_simulate()
# Assert that the user does *not* have admin rights
assert f'jupyter-{username}' not in grp.getgrnam(
'jupyterhub-admins').gr_mem
assert f"jupyter-{username}" not in grp.getgrnam("jupyterhub-admins").gr_mem
@pytest.mark.asyncio
@@ -143,29 +233,37 @@ async def test_long_username():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(32)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
try:
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
# Assert that the user exists
system_username = generate_system_username(f'jupyter-{username}')
system_username = generate_system_username(f"jupyter-{username}")
assert pwd.getpwnam(system_username) is not None
await u.stop_server()
except:
# If we have any errors, print jupyterhub logs before exiting
subprocess.check_call([
'journalctl',
'-u', 'jupyterhub',
'--no-pager'
])
subprocess.check_call(["journalctl", "-u", "jupyterhub", "--no-pager"])
raise
@@ -176,38 +274,56 @@ async def test_user_group_adding():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
groups = {"somegroup": [username]}
# Create the group we want to add the user to
system('groupadd somegroup')
system("groupadd somegroup")
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH,
"add-item",
"users.extra_user_groups.somegroup",
username,
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
try:
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
await u.ensure_server_simulate()
# Assert that the user exists
system_username = generate_system_username(f'jupyter-{username}')
system_username = generate_system_username(f"jupyter-{username}")
assert pwd.getpwnam(system_username) is not None
# Assert that the user was added to the specified group
assert f'jupyter-{username}' in grp.getgrnam('somegroup').gr_mem
assert f"jupyter-{username}" in grp.getgrnam("somegroup").gr_mem
await u.stop_server()
# Delete the group
system('groupdel somegroup')
system("groupdel somegroup")
except:
# If we have any errors, print jupyterhub logs before exiting
subprocess.check_call([
'journalctl',
'-u', 'jupyterhub',
'--no-pager'
])
subprocess.check_call(["journalctl", "-u", "jupyterhub", "--no-pager"])
raise
@@ -219,34 +335,71 @@ async def test_idle_server_culled():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
# Check every 10s for idle servers to cull
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.every", "10"
)
).wait()
)
# Apart from servers, also cull users
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.users", "True"
)
).wait()
)
# Cull servers and users after 60s of activity
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.max_age", "60"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
# Start user's server
await u.ensure_server_simulate()
# Assert that the user exists
assert pwd.getpwnam(f'jupyter-{username}') is not None
assert pwd.getpwnam(f"jupyter-{username}") is not None
# Check that we can get to the user's server
r = await u.session.get(u.hub_url / 'hub/api/users' / username,
headers={'Referer': str(u.hub_url / 'hub/')})
r = await u.session.get(
u.hub_url / "hub/api/users" / username,
headers={"Referer": str(u.hub_url / "hub/")},
)
assert r.status == 200
async def _check_culling_done():
# Check that after 60s, the user and server have been culled and are not reacheable anymore
r = await u.session.get(u.hub_url / 'hub/api/users' / username,
headers={'Referer': str(u.hub_url / 'hub/')})
r = await u.session.get(
u.hub_url / "hub/api/users" / username,
headers={"Referer": str(u.hub_url / "hub/")},
)
print(r.status)
return r.status == 403
@@ -265,34 +418,71 @@ async def test_active_server_not_culled():
"""
# This *must* be localhost, not an IP
# aiohttp throws away cookies if we are connecting to an IP!
hub_url = 'http://localhost'
hub_url = "http://localhost"
username = secrets.token_hex(8)
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "auth.type", "dummy"
)
).wait()
)
# Check every 10s for idle servers to cull
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.every", "10"
)
).wait()
)
# Apart from servers, also cull users
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.users", "True"
)
).wait()
)
# Cull servers and users after 60s of activity
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait()
assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
assert (
0
== await (
await asyncio.create_subprocess_exec(
*TLJH_CONFIG_PATH, "set", "services.cull.max_age", "60"
)
).wait()
)
assert (
0
== await (
await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, "reload")
).wait()
)
async with User(username, hub_url, partial(login_dummy, password='')) as u:
async with User(username, hub_url, partial(login_dummy, password="")) as u:
await u.login()
# Start user's server
await u.ensure_server_simulate()
# Assert that the user exists
assert pwd.getpwnam(f'jupyter-{username}') is not None
assert pwd.getpwnam(f"jupyter-{username}") is not None
# Check that we can get to the user's server
r = await u.session.get(u.hub_url / 'hub/api/users' / username,
headers={'Referer': str(u.hub_url / 'hub/')})
r = await u.session.get(
u.hub_url / "hub/api/users" / username,
headers={"Referer": str(u.hub_url / "hub/")},
)
assert r.status == 200
async def _check_culling_done():
# Check that after 30s, we can still reach the user's server
r = await u.session.get(u.hub_url / 'hub/api/users' / username,
headers={'Referer': str(u.hub_url / 'hub/')})
r = await u.session.get(
u.hub_url / "hub/api/users" / username,
headers={"Referer": str(u.hub_url / "hub/")},
)
print(r.status)
return r.status != 200

15
integration-tests/test_install.py
View File

@@ -119,7 +119,7 @@ def test_admin_writable():
def test_installer_log_readable():
# Test that installer.log is owned by root, and not readable by anyone else
file_stat = os.stat('/opt/tljh/installer.log')
file_stat = os.stat("/opt/tljh/installer.log")
assert file_stat.st_uid == 0
assert file_stat.st_mode == 0o100500
@@ -172,7 +172,16 @@ def test_pip_install(group, allowed):
# get a failure if the user can't install to the global site. Which is
# what we wanted to test for here.
subprocess.check_call(
[python, "-m", "pip", "install", "--no-user", "--ignore-installed", "--no-deps", "flit"],
[
python,
"-m",
"pip",
"install",
"--no-user",
"--ignore-installed",
"--no-deps",
"flit",
],
preexec_fn=partial(setgroup, group),
)
if allowed:
@@ -225,4 +234,4 @@ def test_symlinks():
"""
Test we symlink tljh-config to /usr/local/bin
"""
assert os.path.exists('/usr/bin/tljh-config')
assert os.path.exists("/usr/bin/tljh-config")

3
integration-tests/test_proxy.py
View File

@@ -90,7 +90,6 @@ def test_manual_https(preserve_config):
)
assert resp.code == 200
# cleanup
shutil.rmtree(ssl_dir)
set_config_value(CONFIG_FILE, "https.enabled", False)
@@ -105,7 +104,6 @@ def test_extra_traefik_config():
dynamic_config_dir = os.path.join(STATE_DIR, "rules")
os.makedirs(dynamic_config_dir, exist_ok=True)
extra_static_config = {
"entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}},
"api": {"dashboard": True, "entrypoint": "no_auth_api"},
@@ -126,7 +124,6 @@ def test_extra_traefik_config():
},
}
success = False
for i in range(5):
time.sleep(i)

30
integration-tests/test_simplest_plugin.py
View File

@@ -9,42 +9,30 @@ import subprocess
from tljh.config import CONFIG_FILE, USER_ENV_PREFIX, HUB_ENV_PREFIX
from tljh import user
yaml = YAML(typ='rt')
yaml = YAML(typ="rt")
def test_apt_packages():
"""
Test extra apt packages are installed
"""
assert os.path.exists('/usr/games/sl')
assert os.path.exists("/usr/games/sl")
def test_pip_packages():
"""
Test extra user & hub pip packages are installed
"""
subprocess.check_call([
f'{USER_ENV_PREFIX}/bin/python3',
'-c',
'import django'
])
subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import django"])
subprocess.check_call([
f'{HUB_ENV_PREFIX}/bin/python3',
'-c',
'import there'
])
subprocess.check_call([f"{HUB_ENV_PREFIX}/bin/python3", "-c", "import there"])
def test_conda_packages():
"""
Test extra user conda packages are installed
"""
subprocess.check_call([
f'{USER_ENV_PREFIX}/bin/python3',
'-c',
'import hypothesis'
])
subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import hypothesis"])
def test_config_hook():
@@ -54,16 +42,16 @@ def test_config_hook():
with open(CONFIG_FILE) as f:
data = yaml.load(f)
assert data['simplest_plugin']['present']
assert data["simplest_plugin"]["present"]
def test_jupyterhub_config_hook():
"""
Test that tmpauthenticator is enabled by our custom config plugin
"""
resp = requests.get('http://localhost/hub/tmplogin', allow_redirects=False)
resp = requests.get("http://localhost/hub/tmplogin", allow_redirects=False)
assert resp.status_code == 302
assert resp.headers['Location'] == '/hub/spawn'
assert resp.headers["Location"] == "/hub/spawn"
def test_post_install_hook():
@@ -80,7 +68,7 @@ def test_new_user_create():
"""
Test that plugin receives username as arg
"""
username="user1"
username = "user1"
# Call ensure_user to make sure the user plugin gets called
user.ensure_user(username)

34
setup.py
View File

@@ -1,28 +1,28 @@
from setuptools import setup, find_packages
setup(
name='the-littlest-jupyterhub',
version='0.1',
description='A small JupyterHub distribution',
url='https://github.com/jupyterhub/the-littlest-jupyterhub',
author='Jupyter Development Team',
author_email='jupyter@googlegroups.com',
license='3 Clause BSD',
name="the-littlest-jupyterhub",
version="0.1",
description="A small JupyterHub distribution",
url="https://github.com/jupyterhub/the-littlest-jupyterhub",
author="Jupyter Development Team",
author_email="jupyter@googlegroups.com",
license="3 Clause BSD",
packages=find_packages(),
include_package_data=True,
install_requires=[
'ruamel.yaml==0.17.*',
'jinja2',
'pluggy==1.*',
'passlib',
'backoff',
'requests',
'bcrypt',
'jupyterhub-traefik-proxy==0.3.*',
"ruamel.yaml==0.17.*",
"jinja2",
"pluggy==1.*",
"passlib",
"backoff",
"requests",
"bcrypt",
"jupyterhub-traefik-proxy==0.3.*",
],
entry_points={
'console_scripts': [
'tljh-config = tljh.config:main',
"console_scripts": [
"tljh-config = tljh.config:main",
]
},
)

2
tests/conftest.py
View File

@@ -17,7 +17,7 @@ def tljh_dir(tmpdir):
reload(tljh)
for name in dir(tljh):
mod = getattr(tljh, name)
if isinstance(mod, types.ModuleType) and mod.__name__.startswith('tljh.'):
if isinstance(mod, types.ModuleType) and mod.__name__.startswith("tljh."):
reload(mod)
assert tljh.config.INSTALL_PREFIX == tljh_dir
os.makedirs(tljh.config.STATE_DIR)

56
tests/test_conda.py
View File

@@ -8,24 +8,30 @@ import subprocess
import tempfile
@pytest.fixture(scope='module')
@pytest.fixture(scope="module")
def prefix():
"""
Provide a temporary directory with a mambaforge conda environment
"""
# see https://github.com/conda-forge/miniforge/releases
mambaforge_version = '4.10.3-7'
if os.uname().machine == 'aarch64':
installer_sha256 = "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2"
elif os.uname().machine == 'x86_64':
installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474"
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(v=mambaforge_version, arch=os.uname().machine)
mambaforge_version = "4.10.3-7"
if os.uname().machine == "aarch64":
installer_sha256 = (
"ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2"
)
elif os.uname().machine == "x86_64":
installer_sha256 = (
"fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474"
)
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(
v=mambaforge_version, arch=os.uname().machine
)
with tempfile.TemporaryDirectory() as tmpdir:
with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path:
with conda.download_miniconda_installer(
installer_url, installer_sha256
) as installer_path:
conda.install_miniconda(installer_path, tmpdir)
conda.ensure_conda_packages(tmpdir, [
'conda==4.10.3'
])
conda.ensure_conda_packages(tmpdir, ["conda==4.10.3"])
yield tmpdir
@@ -33,41 +39,29 @@ def test_ensure_packages(prefix):
"""
Test installing packages in conda environment
"""
conda.ensure_conda_packages(prefix, ['numpy'])
conda.ensure_conda_packages(prefix, ["numpy"])
# Throws an error if this fails
subprocess.check_call([
os.path.join(prefix, 'bin', 'python'),
'-c',
'import numpy'
])
subprocess.check_call([os.path.join(prefix, "bin", "python"), "-c", "import numpy"])
def test_ensure_pip_packages(prefix):
"""
Test installing pip packages in conda environment
"""
conda.ensure_conda_packages(prefix, ['pip'])
conda.ensure_pip_packages(prefix, ['numpy'])
conda.ensure_conda_packages(prefix, ["pip"])
conda.ensure_pip_packages(prefix, ["numpy"])
# Throws an error if this fails
subprocess.check_call([
os.path.join(prefix, 'bin', 'python'),
'-c',
'import numpy'
])
subprocess.check_call([os.path.join(prefix, "bin", "python"), "-c", "import numpy"])
def test_ensure_pip_requirements(prefix):
"""
Test installing pip packages with requirements.txt in conda environment
"""
conda.ensure_conda_packages(prefix, ['pip'])
conda.ensure_conda_packages(prefix, ["pip"])
with tempfile.NamedTemporaryFile() as f:
# Sample small package to test
f.write(b'there')
f.write(b"there")
f.flush()
conda.ensure_pip_requirements(prefix, f.name)
subprocess.check_call([
os.path.join(prefix, 'bin', 'python'),
'-c',
'import there'
])
subprocess.check_call([os.path.join(prefix, "bin", "python"), "-c", "import there"])

161
tests/test_config.py
View File

@@ -14,28 +14,25 @@ from tljh import config, configurer
def test_set_no_mutate():
conf = {}
new_conf = config.set_item_in_config(conf, 'a.b', 'c')
assert new_conf['a']['b'] == 'c'
new_conf = config.set_item_in_config(conf, "a.b", "c")
assert new_conf["a"]["b"] == "c"
assert conf == {}
def test_set_one_level():
conf = {}
new_conf = config.set_item_in_config(conf, 'a', 'b')
assert new_conf['a'] == 'b'
new_conf = config.set_item_in_config(conf, "a", "b")
assert new_conf["a"] == "b"
def test_set_multi_level():
conf = {}
new_conf = config.set_item_in_config(conf, 'a.b', 'c')
new_conf = config.set_item_in_config(new_conf, 'a.d', 'e')
new_conf = config.set_item_in_config(new_conf, 'f', 'g')
assert new_conf == {
'a': {'b': 'c', 'd': 'e'},
'f': 'g'
}
new_conf = config.set_item_in_config(conf, "a.b", "c")
new_conf = config.set_item_in_config(new_conf, "a.d", "e")
new_conf = config.set_item_in_config(new_conf, "f", "g")
assert new_conf == {"a": {"b": "c", "d": "e"}, "f": "g"}
def test_set_overwrite():
@@ -44,146 +41,124 @@ def test_set_overwrite():
This might be surprising destructive behavior to some :D
"""
conf = {
'a': 'b'
}
conf = {"a": "b"}
new_conf = config.set_item_in_config(conf, 'a', 'c')
assert new_conf == {'a': 'c'}
new_conf = config.set_item_in_config(conf, "a", "c")
assert new_conf == {"a": "c"}
new_conf = config.set_item_in_config(new_conf, 'a.b', 'd')
assert new_conf == {'a': {'b': 'd'}}
new_conf = config.set_item_in_config(new_conf, "a.b", "d")
assert new_conf == {"a": {"b": "d"}}
new_conf = config.set_item_in_config(new_conf, 'a', 'hi')
assert new_conf == {'a': 'hi'}
new_conf = config.set_item_in_config(new_conf, "a", "hi")
assert new_conf == {"a": "hi"}
def test_unset_no_mutate():
conf = {'a': 'b'}
conf = {"a": "b"}
new_conf = config.unset_item_from_config(conf, 'a')
assert conf == {'a': 'b'}
new_conf = config.unset_item_from_config(conf, "a")
assert conf == {"a": "b"}
def test_unset_one_level():
conf = {'a': 'b'}
conf = {"a": "b"}
new_conf = config.unset_item_from_config(conf, 'a')
new_conf = config.unset_item_from_config(conf, "a")
assert new_conf == {}
def test_unset_multi_level():
conf = {
'a': {'b': 'c', 'd': 'e'},
'f': 'g'
}
conf = {"a": {"b": "c", "d": "e"}, "f": "g"}
new_conf = config.unset_item_from_config(conf, 'a.b')
assert new_conf == {
'a': {'d': 'e'},
'f': 'g'
}
new_conf = config.unset_item_from_config(new_conf, 'a.d')
assert new_conf == {'f': 'g'}
new_conf = config.unset_item_from_config(new_conf, 'f')
new_conf = config.unset_item_from_config(conf, "a.b")
assert new_conf == {"a": {"d": "e"}, "f": "g"}
new_conf = config.unset_item_from_config(new_conf, "a.d")
assert new_conf == {"f": "g"}
new_conf = config.unset_item_from_config(new_conf, "f")
assert new_conf == {}
def test_unset_and_clean_empty_configs():
conf = {
'a': {'b': {'c': {'d': {'e': 'f'}}}}
}
conf = {"a": {"b": {"c": {"d": {"e": "f"}}}}}
new_conf = config.unset_item_from_config(conf, 'a.b.c.d.e')
new_conf = config.unset_item_from_config(conf, "a.b.c.d.e")
assert new_conf == {}
def test_unset_config_error():
with pytest.raises(ValueError):
config.unset_item_from_config({}, 'a')
config.unset_item_from_config({}, "a")
with pytest.raises(ValueError):
config.unset_item_from_config({'a': 'b'}, 'b')
config.unset_item_from_config({"a": "b"}, "b")
with pytest.raises(ValueError):
config.unset_item_from_config({'a': {'b': 'c'}}, 'a.z')
config.unset_item_from_config({"a": {"b": "c"}}, "a.z")
def test_add_to_config_one_level():
conf = {}
new_conf = config.add_item_to_config(conf, 'a.b', 'c')
assert new_conf == {
'a': {'b': ['c']}
}
new_conf = config.add_item_to_config(conf, "a.b", "c")
assert new_conf == {"a": {"b": ["c"]}}
def test_add_to_config_zero_level():
conf = {}
new_conf = config.add_item_to_config(conf, 'a', 'b')
assert new_conf == {
'a': ['b']
}
new_conf = config.add_item_to_config(conf, "a", "b")
assert new_conf == {"a": ["b"]}
def test_add_to_config_multiple():
conf = {}
new_conf = config.add_item_to_config(conf, 'a.b.c', 'd')
assert new_conf == {
'a': {'b': {'c': ['d']}}
}
new_conf = config.add_item_to_config(conf, "a.b.c", "d")
assert new_conf == {"a": {"b": {"c": ["d"]}}}
new_conf = config.add_item_to_config(new_conf, 'a.b.c', 'e')
assert new_conf == {
'a': {'b': {'c': ['d', 'e']}}
}
new_conf = config.add_item_to_config(new_conf, "a.b.c", "e")
assert new_conf == {"a": {"b": {"c": ["d", "e"]}}}
def test_remove_from_config():
conf = {}
new_conf = config.add_item_to_config(conf, 'a.b.c', 'd')
new_conf = config.add_item_to_config(new_conf, 'a.b.c', 'e')
assert new_conf == {
'a': {'b': {'c': ['d', 'e']}}
}
new_conf = config.add_item_to_config(conf, "a.b.c", "d")
new_conf = config.add_item_to_config(new_conf, "a.b.c", "e")
assert new_conf == {"a": {"b": {"c": ["d", "e"]}}}
new_conf = config.remove_item_from_config(new_conf, 'a.b.c', 'e')
assert new_conf == {
'a': {'b': {'c': ['d']}}
}
new_conf = config.remove_item_from_config(new_conf, "a.b.c", "e")
assert new_conf == {"a": {"b": {"c": ["d"]}}}
def test_remove_from_config_error():
with pytest.raises(ValueError):
config.remove_item_from_config({}, 'a.b.c', 'e')
config.remove_item_from_config({}, "a.b.c", "e")
with pytest.raises(ValueError):
config.remove_item_from_config({'a': 'b'}, 'a.b', 'e')
config.remove_item_from_config({"a": "b"}, "a.b", "e")
with pytest.raises(ValueError):
config.remove_item_from_config({'a': ['b']}, 'a', 'e')
config.remove_item_from_config({"a": ["b"]}, "a", "e")
def test_reload_hub():
with mock.patch('tljh.systemd.restart_service') as restart_service, mock.patch(
'tljh.systemd.check_service_active'
) as check_active, mock.patch('tljh.config.check_hub_ready') as check_ready:
config.reload_component('hub')
assert restart_service.called_with('jupyterhub')
assert check_active.called_with('jupyterhub')
with mock.patch("tljh.systemd.restart_service") as restart_service, mock.patch(
"tljh.systemd.check_service_active"
) as check_active, mock.patch("tljh.config.check_hub_ready") as check_ready:
config.reload_component("hub")
assert restart_service.called_with("jupyterhub")
assert check_active.called_with("jupyterhub")
def test_reload_proxy(tljh_dir):
with mock.patch("tljh.systemd.restart_service") as restart_service, mock.patch(
"tljh.systemd.check_service_active"
) as check_active:
config.reload_component('proxy')
assert restart_service.called_with('traefik')
assert check_active.called_with('traefik')
assert os.path.exists(os.path.join(config.STATE_DIR, 'traefik.toml'))
config.reload_component("proxy")
assert restart_service.called_with("traefik")
assert check_active.called_with("traefik")
assert os.path.exists(os.path.join(config.STATE_DIR, "traefik.toml"))
def test_cli_no_command(capsys):
@@ -193,51 +168,45 @@ def test_cli_no_command(capsys):
assert "positional arguments:" in captured.out
@pytest.mark.parametrize(
"arg, value",
[
("true", True),
("FALSE", False)
]
)
@pytest.mark.parametrize("arg, value", [("true", True), ("FALSE", False)])
def test_cli_set_bool(tljh_dir, arg, value):
config.main(["set", "https.enabled", arg])
cfg = configurer.load_config()
assert cfg['https']['enabled'] == value
assert cfg["https"]["enabled"] == value
def test_cli_set_int(tljh_dir):
config.main(["set", "https.port", "123"])
cfg = configurer.load_config()
assert cfg['https']['port'] == 123
assert cfg["https"]["port"] == 123
def test_cli_unset(tljh_dir):
config.main(["set", "foo.bar", "1"])
config.main(["set", "foo.bar2", "2"])
cfg = configurer.load_config()
assert cfg['foo'] == {'bar': 1, 'bar2': 2}
assert cfg["foo"] == {"bar": 1, "bar2": 2}
config.main(["unset", "foo.bar"])
cfg = configurer.load_config()
assert cfg['foo'] == {'bar2': 2}
assert cfg["foo"] == {"bar2": 2}
def test_cli_add_float(tljh_dir):
config.main(["add-item", "foo.bar", "1.25"])
cfg = configurer.load_config()
assert cfg['foo']['bar'] == [1.25]
assert cfg["foo"]["bar"] == [1.25]
def test_cli_remove_int(tljh_dir):
config.main(["add-item", "foo.bar", "1"])
config.main(["add-item", "foo.bar", "2"])
cfg = configurer.load_config()
assert cfg['foo']['bar'] == [1, 2]
assert cfg["foo"]["bar"] == [1, 2]
config.main(["remove-item", "foo.bar", "1"])
cfg = configurer.load_config()
assert cfg['foo']['bar'] == [2]
assert cfg["foo"]["bar"] == [2]
@pytest.mark.parametrize(

194
tests/test_configurer.py
View File

@@ -9,9 +9,6 @@ import sys
from tljh import configurer
def apply_mock_config(overrides):
"""
Configure a mock configurer with given overrides.
@@ -28,15 +25,15 @@ def test_default_base_url():
Test default JupyterHub base_url
"""
c = apply_mock_config({})
assert c.JupyterHub.base_url == '/'
assert c.JupyterHub.base_url == "/"
def test_set_base_url():
"""
Test set JupyterHub base_url
"""
c = apply_mock_config({'base_url': '/custom-base'})
assert c.JupyterHub.base_url == '/custom-base'
c = apply_mock_config({"base_url": "/custom-base"})
assert c.JupyterHub.base_url == "/custom-base"
def test_default_memory_limit():
@@ -51,8 +48,8 @@ def test_set_memory_limit():
"""
Test setting per user memory limit
"""
c = apply_mock_config({'limits': {'memory': '42G'}})
assert c.Spawner.mem_limit == '42G'
c = apply_mock_config({"limits": {"memory": "42G"}})
assert c.Spawner.mem_limit == "42G"
def test_app_default():
@@ -61,23 +58,23 @@ def test_app_default():
"""
c = apply_mock_config({})
# default_url is not set, so JupyterHub will pick default.
assert 'default_url' not in c.Spawner
assert "default_url" not in c.Spawner
def test_app_jupyterlab():
"""
Test setting JupyterLab as default application
"""
c = apply_mock_config({'user_environment': {'default_app': 'jupyterlab'}})
assert c.Spawner.default_url == '/lab'
c = apply_mock_config({"user_environment": {"default_app": "jupyterlab"}})
assert c.Spawner.default_url == "/lab"
def test_app_nteract():
"""
Test setting nteract as default application
"""
c = apply_mock_config({'user_environment': {'default_app': 'nteract'}})
assert c.Spawner.default_url == '/nteract'
c = apply_mock_config({"user_environment": {"default_app": "nteract"}})
assert c.Spawner.default_url == "/nteract"
def test_auth_default():
@@ -86,7 +83,10 @@ def test_auth_default():
"""
c = apply_mock_config({})
assert c.JupyterHub.authenticator_class == 'firstuseauthenticator.FirstUseAuthenticator'
assert (
c.JupyterHub.authenticator_class
== "firstuseauthenticator.FirstUseAuthenticator"
)
# Do not auto create users who haven't been manually added by default
assert not c.FirstUseAuthenticator.create_users
@@ -95,49 +95,43 @@ def test_auth_dummy():
"""
Test setting Dummy Authenticator & password
"""
c = apply_mock_config({
'auth': {
'type': 'dummy',
'DummyAuthenticator': {
'password': 'test'
}
}
})
assert c.JupyterHub.authenticator_class == 'dummy'
assert c.DummyAuthenticator.password == 'test'
c = apply_mock_config(
{"auth": {"type": "dummy", "DummyAuthenticator": {"password": "test"}}}
)
assert c.JupyterHub.authenticator_class == "dummy"
assert c.DummyAuthenticator.password == "test"
def test_user_groups():
"""
Test setting user groups
"""
c = apply_mock_config({
'users': {
'extra_user_groups': {
"g1": ["u1", "u2"],
"g2": ["u3", "u4"]
},
c = apply_mock_config(
{
"users": {
"extra_user_groups": {"g1": ["u1", "u2"], "g2": ["u3", "u4"]},
}
}
})
assert c.UserCreatingSpawner.user_groups == {
"g1": ["u1", "u2"],
"g2": ["u3", "u4"]
}
)
assert c.UserCreatingSpawner.user_groups == {"g1": ["u1", "u2"], "g2": ["u3", "u4"]}
def test_auth_firstuse():
"""
Test setting FirstUse Authenticator options
"""
c = apply_mock_config({
'auth': {
'type': 'firstuseauthenticator.FirstUseAuthenticator',
'FirstUseAuthenticator': {
'create_users': True
c = apply_mock_config(
{
"auth": {
"type": "firstuseauthenticator.FirstUseAuthenticator",
"FirstUseAuthenticator": {"create_users": True},
}
}
})
assert c.JupyterHub.authenticator_class == 'firstuseauthenticator.FirstUseAuthenticator'
)
assert (
c.JupyterHub.authenticator_class
== "firstuseauthenticator.FirstUseAuthenticator"
)
assert c.FirstUseAuthenticator.create_users
@@ -145,18 +139,22 @@ def test_auth_github():
"""
Test using GitHub authenticator
"""
c = apply_mock_config({
'auth': {
'type': 'oauthenticator.github.GitHubOAuthenticator',
'GitHubOAuthenticator': {
'client_id': 'something',
'client_secret': 'something-else'
c = apply_mock_config(
{
"auth": {
"type": "oauthenticator.github.GitHubOAuthenticator",
"GitHubOAuthenticator": {
"client_id": "something",
"client_secret": "something-else",
},
}
}
})
assert c.JupyterHub.authenticator_class == 'oauthenticator.github.GitHubOAuthenticator'
assert c.GitHubOAuthenticator.client_id == 'something'
assert c.GitHubOAuthenticator.client_secret == 'something-else'
)
assert (
c.JupyterHub.authenticator_class == "oauthenticator.github.GitHubOAuthenticator"
)
assert c.GitHubOAuthenticator.client_id == "something"
assert c.GitHubOAuthenticator.client_secret == "something-else"
def test_traefik_api_default():
@@ -165,7 +163,7 @@ def test_traefik_api_default():
"""
c = apply_mock_config({})
assert c.TraefikTomlProxy.traefik_api_username == 'api_admin'
assert c.TraefikTomlProxy.traefik_api_username == "api_admin"
assert len(c.TraefikTomlProxy.traefik_api_password) == 0
@@ -173,14 +171,11 @@ def test_set_traefik_api():
"""
Test setting per traefik api credentials
"""
c = apply_mock_config({
'traefik_api': {
'username': 'some_user',
'password': '1234'
}
})
assert c.TraefikTomlProxy.traefik_api_username == 'some_user'
assert c.TraefikTomlProxy.traefik_api_password == '1234'
c = apply_mock_config(
{"traefik_api": {"username": "some_user", "password": "1234"}}
)
assert c.TraefikTomlProxy.traefik_api_username == "some_user"
assert c.TraefikTomlProxy.traefik_api_password == "1234"
def test_cull_service_default():
@@ -190,51 +185,58 @@ def test_cull_service_default():
c = apply_mock_config({})
cull_cmd = [
sys.executable, '-m', 'jupyterhub_idle_culler',
'--timeout=600', '--cull-every=60', '--concurrency=5',
'--max-age=0'
sys.executable,
"-m",
"jupyterhub_idle_culler",
"--timeout=600",
"--cull-every=60",
"--concurrency=5",
"--max-age=0",
]
assert c.JupyterHub.services == [
{
"name": "cull-idle",
"admin": True,
"command": cull_cmd,
}
]
assert c.JupyterHub.services == [{
'name': 'cull-idle',
'admin': True,
'command': cull_cmd,
}]
def test_set_cull_service():
"""
Test setting cull service options
"""
c = apply_mock_config({
'services': {
'cull': {
'every': 10,
'users': True,
'max_age': 60
}
}
})
c = apply_mock_config(
{"services": {"cull": {"every": 10, "users": True, "max_age": 60}}}
)
cull_cmd = [
sys.executable, '-m', 'jupyterhub_idle_culler',
'--timeout=600', '--cull-every=10', '--concurrency=5',
'--max-age=60', '--cull-users'
sys.executable,
"-m",
"jupyterhub_idle_culler",
"--timeout=600",
"--cull-every=10",
"--concurrency=5",
"--max-age=60",
"--cull-users",
]
assert c.JupyterHub.services == [
{
"name": "cull-idle",
"admin": True,
"command": cull_cmd,
}
]
assert c.JupyterHub.services == [{
'name': 'cull-idle',
'admin': True,
'command': cull_cmd,
}]
def test_load_secrets(tljh_dir):
"""
Test loading secret files
"""
with open(os.path.join(tljh_dir, 'state', 'traefik-api.secret'), 'w') as f:
with open(os.path.join(tljh_dir, "state", "traefik-api.secret"), "w") as f:
f.write("traefik-password")
tljh_config = configurer.load_config()
assert tljh_config['traefik_api']['password'] == "traefik-password"
assert tljh_config["traefik_api"]["password"] == "traefik-password"
c = apply_mock_config(tljh_config)
assert c.TraefikTomlProxy.traefik_api_password == "traefik-password"
@@ -243,13 +245,15 @@ def test_auth_native():
"""
Test setting Native Authenticator
"""
c = apply_mock_config({
'auth': {
'type': 'nativeauthenticator.NativeAuthenticator',
'NativeAuthenticator': {
'open_signup': True,
c = apply_mock_config(
{
"auth": {
"type": "nativeauthenticator.NativeAuthenticator",
"NativeAuthenticator": {
"open_signup": True,
},
}
}
})
assert c.JupyterHub.authenticator_class == 'nativeauthenticator.NativeAuthenticator'
)
assert c.JupyterHub.authenticator_class == "nativeauthenticator.NativeAuthenticator"
assert c.NativeAuthenticator.open_signup == True

30
tests/test_installer.py
View File

@@ -13,26 +13,26 @@ def test_ensure_config_yaml(tljh_dir):
installer.ensure_config_yaml(pm)
assert os.path.exists(installer.CONFIG_FILE)
assert os.path.isdir(installer.CONFIG_DIR)
assert os.path.isdir(os.path.join(installer.CONFIG_DIR, 'jupyterhub_config.d'))
assert os.path.isdir(os.path.join(installer.CONFIG_DIR, "jupyterhub_config.d"))
# verify that old config doesn't exist
assert not os.path.exists(os.path.join(tljh_dir, 'config.yaml'))
assert not os.path.exists(os.path.join(tljh_dir, "config.yaml"))
@pytest.mark.parametrize(
"admins, expected_config",
[
([['a1'], ['a2'], ['a3']], ['a1', 'a2', 'a3']),
([['a1:p1'], ['a2']], ['a1', 'a2']),
],
"admins, expected_config",
[
([["a1"], ["a2"], ["a3"]], ["a1", "a2", "a3"]),
([["a1:p1"], ["a2"]], ["a1", "a2"]),
],
)
def test_ensure_admins(tljh_dir, admins, expected_config):
# --admin option called multiple times on the installer
# creates a list of argument lists.
installer.ensure_admins(admins)
# --admin option called multiple times on the installer
# creates a list of argument lists.
installer.ensure_admins(admins)
config_path = installer.CONFIG_FILE
with open(config_path) as f:
config = yaml.load(f)
config_path = installer.CONFIG_FILE
with open(config_path) as f:
config = yaml.load(f)
# verify the list was flattened
assert config['users']['admin'] == expected_config
# verify the list was flattened
assert config["users"]["admin"] == expected_config

9
tests/test_normalize.py
View File

@@ -10,14 +10,13 @@ def test_generate_username():
"""
usernames = {
# Very short
'jupyter-test': 'jupyter-test',
"jupyter-test": "jupyter-test",
# Very long
'jupyter-aelie9sohjeequ9iemeipuimuoshahz4aitugiuteeg4ohioh5yuiha6aei7te5z': 'jupyter-aelie9sohjeequ9iem-4b726',
"jupyter-aelie9sohjeequ9iemeipuimuoshahz4aitugiuteeg4ohioh5yuiha6aei7te5z": "jupyter-aelie9sohjeequ9iem-4b726",
# 26 characters, just below our cutoff for hashing
'jupyter-abcdefghijklmnopq': 'jupyter-abcdefghijklmnopq',
"jupyter-abcdefghijklmnopq": "jupyter-abcdefghijklmnopq",
# 27 characters, just above our cutoff for hashing
'jupyter-abcdefghijklmnopqr': 'jupyter-abcdefghijklmnopqr-e375e',
"jupyter-abcdefghijklmnopqr": "jupyter-abcdefghijklmnopqr-e375e",
}
for hub_user, system_user in usernames.items():
assert generate_system_username(hub_user) == system_user

29
tests/test_traefik.py
View File

@@ -36,10 +36,8 @@ def test_default_config(tmpdir, tljh_dir):
"http": {"address": ":80"},
"auth_api": {
"address": "127.0.0.1:8099",
"auth": {
"basic": {"users": [""]}
},
"whiteList": {"sourceRange": ["127.0.0.1"]}
"auth": {"basic": {"users": [""]}},
"whiteList": {"sourceRange": ["127.0.0.1"]},
},
}
@@ -72,10 +70,8 @@ def test_letsencrypt_config(tljh_dir):
"https": {"address": ":443", "tls": {"minVersion": "VersionTLS12"}},
"auth_api": {
"address": "127.0.0.1:8099",
"auth": {
"basic": {"users": [""]}
},
"whiteList": {"sourceRange": ["127.0.0.1"]}
"auth": {"basic": {"users": [""]}},
"whiteList": {"sourceRange": ["127.0.0.1"]},
},
}
assert cfg["acme"] == {
@@ -113,18 +109,17 @@ def test_manual_ssl_config(tljh_dir):
"minVersion": "VersionTLS12",
"certificates": [
{"certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key"}
]
],
},
},
"auth_api": {
"address": "127.0.0.1:8099",
"auth": {
"basic": {"users": [""]}
},
"whiteList": {"sourceRange": ["127.0.0.1"]}
"auth": {"basic": {"users": [""]}},
"whiteList": {"sourceRange": ["127.0.0.1"]},
},
}
def test_extra_config(tmpdir, tljh_dir):
extra_config_dir = os.path.join(tljh_dir, config.CONFIG_DIR, "traefik_config.d")
state_dir = tmpdir.mkdir("state")
@@ -146,13 +141,9 @@ def test_extra_config(tmpdir, tljh_dir):
# modify existing value
"logLevel": "ERROR",
# modify existing value with multiple levels
"entryPoints": {
"auth_api": {
"address": "127.0.0.1:9999"
}
},
"entryPoints": {"auth_api": {"address": "127.0.0.1:9999"}},
# add new setting
"checkNewVersion": False
"checkNewVersion": False,
}
with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file:

20
tests/test_user.py
View File

@@ -16,7 +16,7 @@ def test_ensure_user():
Test user creation & removal
"""
# Use a prefix to make sure we never start with a number
username = 'u' + str(uuid.uuid4())[:8]
username = "u" + str(uuid.uuid4())[:8]
# Validate that no user exists
with pytest.raises(KeyError):
pwd.getpwnam(username)
@@ -31,9 +31,15 @@ def test_ensure_user():
assert os.path.exists(home_dir)
# Ensure not word readable/writable especially in teaching context
homedir_stats = os.stat(home_dir).st_mode
assert not (homedir_stats & stat.S_IROTH), "Everyone should not be able to read users home directory"
assert not (homedir_stats & stat.S_IWOTH), "Everyone should not be able to write users home directory"
assert not (homedir_stats & stat.S_IXOTH), "Everyone should not be able to list what is in users home directory"
assert not (
homedir_stats & stat.S_IROTH
), "Everyone should not be able to read users home directory"
assert not (
homedir_stats & stat.S_IWOTH
), "Everyone should not be able to write users home directory"
assert not (
homedir_stats & stat.S_IXOTH
), "Everyone should not be able to list what is in users home directory"
# Run ensure_user again, should be a noop
user.ensure_user(username)
@@ -51,7 +57,7 @@ def test_ensure_group():
Test group creation & removal
"""
# Use a prefix to make sure we never start with a number
groupname = 'g' + str(uuid.uuid4())[:8]
groupname = "g" + str(uuid.uuid4())[:8]
# Validate that no group exists
with pytest.raises(KeyError):
@@ -77,8 +83,8 @@ def test_group_membership():
"""
Test group memberships can be added / removed
"""
username = 'u' + str(uuid.uuid4())[:8]
groupname = 'g' + str(uuid.uuid4())[:8]
username = "u" + str(uuid.uuid4())[:8]
groupname = "g" + str(uuid.uuid4())[:8]
# Validate that no group exists
with pytest.raises(KeyError):

18
tests/test_utils.py
View File

@@ -5,17 +5,15 @@ import logging
def test_run_subprocess_exception(mocker):
logger = logging.getLogger('tljh')
mocker.patch.object(logger, 'error')
logger = logging.getLogger("tljh")
mocker.patch.object(logger, "error")
with pytest.raises(subprocess.CalledProcessError):
utils.run_subprocess(
['/bin/bash', '-c', 'echo error; exit 1']
)
logger.error.assert_called_with('error\n')
utils.run_subprocess(["/bin/bash", "-c", "echo error; exit 1"])
logger.error.assert_called_with("error\n")
def test_run_subprocess(mocker):
logger = logging.getLogger('tljh')
mocker.patch.object(logger, 'debug')
utils.run_subprocess(['/bin/bash', '-c', 'echo success'])
logger.debug.assert_called_with('success\n')
logger = logging.getLogger("tljh")
mocker.patch.object(logger, "debug")
utils.run_subprocess(["/bin/bash", "-c", "echo success"])
logger.debug.assert_called_with("success\n")

33
tljh/apt.py
View File

@@ -13,9 +13,9 @@ def trust_gpg_key(key):
key is a GPG public key (bytes) that can be passed to apt-key add via stdin.
"""
# If gpg2 doesn't exist, install it.
if not os.path.exists('/usr/bin/gpg2'):
install_packages(['gnupg2'])
utils.run_subprocess(['apt-key', 'add', '-'], input=key)
if not os.path.exists("/usr/bin/gpg2"):
install_packages(["gnupg2"])
utils.run_subprocess(["apt-key", "add", "-"], input=key)
def add_source(name, source_url, section):
@@ -25,15 +25,22 @@ def add_source(name, source_url, section):
distro is determined from /etc/os-release
"""
# lsb_release is not installed in most docker images by default
distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}'], stderr=subprocess.STDOUT).decode().strip()
line = f'deb {source_url} {distro} {section}\n'
with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f:
distro = (
subprocess.check_output(
["/bin/bash", "-c", "source /etc/os-release && echo ${VERSION_CODENAME}"],
stderr=subprocess.STDOUT,
)
.decode()
.strip()
)
line = f"deb {source_url} {distro} {section}\n"
with open(os.path.join("/etc/apt/sources.list.d/", name + ".list"), "a+") as f:
# Write out deb line only if it already doesn't exist
f.seek(0)
if line not in f.read():
f.write(line)
f.truncate()
utils.run_subprocess(['apt-get', 'update', '--yes'])
utils.run_subprocess(["apt-get", "update", "--yes"])
def install_packages(packages):
@@ -41,13 +48,9 @@ def install_packages(packages):
Install debian packages
"""
# Check if an apt-get update is required
if len(os.listdir('/var/lib/apt/lists')) == 0:
utils.run_subprocess(['apt-get', 'update', '--yes'])
if len(os.listdir("/var/lib/apt/lists")) == 0:
utils.run_subprocess(["apt-get", "update", "--yes"])
env = os.environ.copy()
# Stop apt from asking questions!
env['DEBIAN_FRONTEND'] = 'noninteractive'
utils.run_subprocess([
'apt-get',
'install',
'--yes'
] + packages, env=env)
env["DEBIAN_FRONTEND"] = "noninteractive"
utils.run_subprocess(["apt-get", "install", "--yes"] + packages, env=env)

80
tljh/conda.py
View File

@@ -30,10 +30,14 @@ def check_miniconda_version(prefix, version):
Return true if a miniconda install with version exists at prefix
"""
try:
installed_version = subprocess.check_output([
os.path.join(prefix, 'bin', 'conda'),
'-V'
], stderr=subprocess.STDOUT).decode().strip().split()[1]
installed_version = (
subprocess.check_output(
[os.path.join(prefix, "bin", "conda"), "-V"], stderr=subprocess.STDOUT
)
.decode()
.strip()
.split()[1]
)
return V(installed_version) >= V(version)
except (subprocess.CalledProcessError, FileNotFoundError):
# Conda doesn't exist
@@ -49,7 +53,7 @@ def download_miniconda_installer(installer_url, sha256sum):
of given version, verifies the sha256sum & provides path to it to the `with`
block to run.
"""
with tempfile.NamedTemporaryFile('wb') as f:
with tempfile.NamedTemporaryFile("wb") as f:
f.write(requests.get(installer_url).content)
# Remain in the NamedTemporaryFile context, but flush changes, see:
# https://docs.python.org/3/library/os.html#os.fsync
@@ -57,7 +61,7 @@ def download_miniconda_installer(installer_url, sha256sum):
os.fsync(f.fileno())
if sha256_file(f.name) != sha256sum:
raise Exception('sha256sum hash mismatch! Downloaded file corrupted')
raise Exception("sha256sum hash mismatch! Downloaded file corrupted")
yield f.name
@@ -71,9 +75,7 @@ def fix_permissions(prefix):
Run after each install command.
"""
utils.run_subprocess(
["chown", "-R", f"{os.getuid()}:{os.getgid()}", prefix]
)
utils.run_subprocess(["chown", "-R", f"{os.getuid()}:{os.getgid()}", prefix])
utils.run_subprocess(["chmod", "-R", "o-w", prefix])
@@ -81,12 +83,7 @@ def install_miniconda(installer_path, prefix):
"""
Install miniconda with installer at installer_path under prefix
"""
utils.run_subprocess([
'/bin/bash',
installer_path,
'-u', '-b',
'-p', prefix
])
utils.run_subprocess(["/bin/bash", installer_path, "-u", "-b", "-p", prefix])
# fix permissions on initial install
# a few files have the wrong ownership and permissions initially
# when the installer is run as root
@@ -100,29 +97,38 @@ def ensure_conda_packages(prefix, packages):
Note that conda seem to update dependencies by default, so there is probably
no need to have a update parameter exposed for this function.
"""
conda_executable = [os.path.join(prefix, 'bin', 'mamba')]
conda_executable = [os.path.join(prefix, "bin", "mamba")]
abspath = os.path.abspath(prefix)
# Let subprocess errors propagate
# Explicitly do *not* capture stderr, since that's not always JSON!
# Scripting conda is a PITA!
# FIXME: raise different exception when using
raw_output = subprocess.check_output(conda_executable + [
'install',
'-c', 'conda-forge', # Make customizable if we ever need to
'--json',
'--prefix', abspath
] + packages).decode()
raw_output = subprocess.check_output(
conda_executable
+ [
"install",
"-c",
"conda-forge", # Make customizable if we ever need to
"--json",
"--prefix",
abspath,
]
+ packages
).decode()
# `conda install` outputs JSON lines for fetch updates,
# and a undelimited output at the end. There is no reasonable way to
# parse this outside of this kludge.
filtered_output = '\n'.join([
l for l in raw_output.split('\n')
# Sometimes the JSON messages start with a \x00. The lstrip removes these.
# conda messages seem to randomly throw \x00 in places for no reason
if not l.lstrip('\x00').startswith('{"fetch"')
])
output = json.loads(filtered_output.lstrip('\x00'))
if 'success' in output and output['success'] == True:
filtered_output = "\n".join(
[
l
for l in raw_output.split("\n")
# Sometimes the JSON messages start with a \x00. The lstrip removes these.
# conda messages seem to randomly throw \x00 in places for no reason
if not l.lstrip("\x00").startswith('{"fetch"')
]
)
output = json.loads(filtered_output.lstrip("\x00"))
if "success" in output and output["success"] == True:
return
fix_permissions(prefix)
@@ -132,10 +138,10 @@ def ensure_pip_packages(prefix, packages, upgrade=False):
Ensure pip packages are installed in the given conda prefix.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
pip_cmd = pip_executable + ['install']
pip_executable = [os.path.join(abspath, "bin", "python"), "-m", "pip"]
pip_cmd = pip_executable + ["install"]
if upgrade:
pip_cmd.append('--upgrade')
pip_cmd.append("--upgrade")
utils.run_subprocess(pip_cmd + packages)
fix_permissions(prefix)
@@ -147,9 +153,9 @@ def ensure_pip_requirements(prefix, requirements_path, upgrade=False):
requirements_path can be a file or a URL.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
pip_cmd = pip_executable + ['install']
pip_executable = [os.path.join(abspath, "bin", "python"), "-m", "pip"]
pip_cmd = pip_executable + ["install"]
if upgrade:
pip_cmd.append('--upgrade')
utils.run_subprocess(pip_cmd + ['--requirement', requirements_path])
pip_cmd.append("--upgrade")
utils.run_subprocess(pip_cmd + ["--requirement", requirements_path])
fix_permissions(prefix)

166
tljh/config.py
View File

@@ -25,12 +25,12 @@ import requests
from .yaml import yaml
INSTALL_PREFIX = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'hub')
USER_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'user')
STATE_DIR = os.path.join(INSTALL_PREFIX, 'state')
CONFIG_DIR = os.path.join(INSTALL_PREFIX, 'config')
CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.yaml')
INSTALL_PREFIX = os.environ.get("TLJH_INSTALL_PREFIX", "/opt/tljh")
HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, "hub")
USER_ENV_PREFIX = os.path.join(INSTALL_PREFIX, "user")
STATE_DIR = os.path.join(INSTALL_PREFIX, "state")
CONFIG_DIR = os.path.join(INSTALL_PREFIX, "config")
CONFIG_FILE = os.path.join(CONFIG_DIR, "config.yaml")
def set_item_in_config(config, property_path, value):
@@ -42,7 +42,7 @@ def set_item_in_config(config, property_path, value):
property_path is a series of dot separated values. Any part of the path
that does not exist is created.
"""
path_components = property_path.split('.')
path_components = property_path.split(".")
# Mutate a copy of the config, not config itself
cur_part = config_copy = deepcopy(config)
@@ -69,7 +69,7 @@ def unset_item_from_config(config, property_path):
property_path is a series of dot separated values.
"""
path_components = property_path.split('.')
path_components = property_path.split(".")
# Mutate a copy of the config, not config itself
cur_part = config_copy = deepcopy(config)
@@ -94,13 +94,13 @@ def unset_item_from_config(config, property_path):
for i, cur_path in enumerate(path_components):
if i == len(path_components) - 1:
if cur_path not in cur_part:
raise ValueError(f'{property_path} does not exist in config!')
raise ValueError(f"{property_path} does not exist in config!")
del cur_part[cur_path]
remove_empty_configs(config_copy, path_components[:-1])
break
else:
if cur_path not in cur_part:
raise ValueError(f'{property_path} does not exist in config!')
raise ValueError(f"{property_path} does not exist in config!")
cur_part = cur_part[cur_path]
return config_copy
@@ -110,7 +110,7 @@ def add_item_to_config(config, property_path, value):
"""
Add an item to a list in config.
"""
path_components = property_path.split('.')
path_components = property_path.split(".")
# Mutate a copy of the config, not config itself
cur_part = config_copy = deepcopy(config)
@@ -136,7 +136,7 @@ def remove_item_from_config(config, property_path, value):
"""
Remove an item from a list in config.
"""
path_components = property_path.split('.')
path_components = property_path.split(".")
# Mutate a copy of the config, not config itself
cur_part = config_copy = deepcopy(config)
@@ -144,12 +144,12 @@ def remove_item_from_config(config, property_path, value):
if i == len(path_components) - 1:
# Final component, it must be a list and we delete from it
if cur_path not in cur_part or not _is_list(cur_part[cur_path]):
raise ValueError(f'{property_path} is not a list')
raise ValueError(f"{property_path} is not a list")
cur_part = cur_part[cur_path]
cur_part.remove(value)
else:
if cur_path not in cur_part or not _is_dict(cur_part[cur_path]):
raise ValueError(f'{property_path} does not exist in config!')
raise ValueError(f"{property_path} does not exist in config!")
cur_part = cur_part[cur_path]
return config_copy
@@ -182,7 +182,7 @@ def set_config_value(config_path, key_path, value):
config = set_item_in_config(config, key_path, value)
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
yaml.dump(config, f)
@@ -200,7 +200,7 @@ def unset_config_value(config_path, key_path):
config = unset_item_from_config(config, key_path)
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
yaml.dump(config, f)
@@ -218,7 +218,7 @@ def add_config_value(config_path, key_path, value):
config = add_item_to_config(config, key_path, value)
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
yaml.dump(config, f)
@@ -236,18 +236,20 @@ def remove_config_value(config_path, key_path, value):
config = remove_item_from_config(config, key_path, value)
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
yaml.dump(config, f)
def check_hub_ready():
from .configurer import load_config
base_url = load_config()['base_url']
base_url = base_url[:-1] if base_url[-1] == '/' else base_url
http_port = load_config()['http']['port']
base_url = load_config()["base_url"]
base_url = base_url[:-1] if base_url[-1] == "/" else base_url
http_port = load_config()["http"]["port"]
try:
r = requests.get('http://127.0.0.1:%d%s/hub/api' % (http_port, base_url), verify=False)
r = requests.get(
"http://127.0.0.1:%d%s/hub/api" % (http_port, base_url), verify=False
)
return r.status_code == 200
except:
return False
@@ -262,33 +264,33 @@ def reload_component(component):
# import here to avoid circular imports
from tljh import systemd, traefik
if component == 'hub':
systemd.restart_service('jupyterhub')
if component == "hub":
systemd.restart_service("jupyterhub")
# Ensure hub is back up
while not systemd.check_service_active('jupyterhub'):
while not systemd.check_service_active("jupyterhub"):
time.sleep(1)
while not check_hub_ready():
time.sleep(1)
print('Hub reload with new configuration complete')
elif component == 'proxy':
print("Hub reload with new configuration complete")
elif component == "proxy":
traefik.ensure_traefik_config(STATE_DIR)
systemd.restart_service('traefik')
while not systemd.check_service_active('traefik'):
systemd.restart_service("traefik")
while not systemd.check_service_active("traefik"):
time.sleep(1)
print('Proxy reload with new configuration complete')
print("Proxy reload with new configuration complete")
def parse_value(value_str):
"""Parse a value string"""
if value_str is None:
return value_str
if re.match(r'^\d+$', value_str):
if re.match(r"^\d+$", value_str):
return int(value_str)
elif re.match(r'^\d+\.\d*$', value_str):
elif re.match(r"^\d+\.\d*$", value_str):
return float(value_str)
elif value_str.lower() == 'true':
elif value_str.lower() == "true":
return True
elif value_str.lower() == 'false':
elif value_str.lower() == "false":
return False
else:
# it's a string
@@ -306,13 +308,17 @@ def _is_list(item):
def main(argv=None):
if os.geteuid() != 0:
print("tljh-config needs root privileges to run", file=sys.stderr)
print("Try using sudo before the tljh-config command you wanted to run", file=sys.stderr)
print(
"Try using sudo before the tljh-config command you wanted to run",
file=sys.stderr,
)
sys.exit(1)
if argv is None:
argv = sys.argv[1:]
from .log import init_logging
try:
init_logging()
except Exception as e:
@@ -321,97 +327,67 @@ def main(argv=None):
argparser = argparse.ArgumentParser()
argparser.add_argument(
'--config-path',
default=CONFIG_FILE,
help='Path to TLJH config.yaml file'
"--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file"
)
subparsers = argparser.add_subparsers(dest='action')
subparsers = argparser.add_subparsers(dest="action")
show_parser = subparsers.add_parser(
'show',
help='Show current configuration'
)
show_parser = subparsers.add_parser("show", help="Show current configuration")
unset_parser = subparsers.add_parser(
'unset',
help='Unset a configuration property'
)
unset_parser = subparsers.add_parser("unset", help="Unset a configuration property")
unset_parser.add_argument(
'key_path',
help='Dot separated path to configuration key to unset'
"key_path", help="Dot separated path to configuration key to unset"
)
set_parser = subparsers.add_parser(
'set',
help='Set a configuration property'
)
set_parser = subparsers.add_parser("set", help="Set a configuration property")
set_parser.add_argument(
'key_path',
help='Dot separated path to configuration key to set'
)
set_parser.add_argument(
'value',
help='Value to set the configuration key to'
"key_path", help="Dot separated path to configuration key to set"
)
set_parser.add_argument("value", help="Value to set the configuration key to")
add_item_parser = subparsers.add_parser(
'add-item',
help='Add a value to a list for a configuration property'
"add-item", help="Add a value to a list for a configuration property"
)
add_item_parser.add_argument(
'key_path',
help='Dot separated path to configuration key to add value to'
)
add_item_parser.add_argument(
'value',
help='Value to add to the configuration key'
"key_path", help="Dot separated path to configuration key to add value to"
)
add_item_parser.add_argument("value", help="Value to add to the configuration key")
remove_item_parser = subparsers.add_parser(
'remove-item',
help='Remove a value from a list for a configuration property'
"remove-item", help="Remove a value from a list for a configuration property"
)
remove_item_parser.add_argument(
'key_path',
help='Dot separated path to configuration key to remove value from'
)
remove_item_parser.add_argument(
'value',
help='Value to remove from key_path'
"key_path", help="Dot separated path to configuration key to remove value from"
)
remove_item_parser.add_argument("value", help="Value to remove from key_path")
reload_parser = subparsers.add_parser(
'reload',
help='Reload a component to apply configuration change'
"reload", help="Reload a component to apply configuration change"
)
reload_parser.add_argument(
'component',
choices=('hub', 'proxy'),
help='Which component to reload',
default='hub',
nargs='?'
"component",
choices=("hub", "proxy"),
help="Which component to reload",
default="hub",
nargs="?",
)
args = argparser.parse_args(argv)
if args.action == 'show':
if args.action == "show":
show_config(args.config_path)
elif args.action == 'set':
set_config_value(args.config_path, args.key_path,
parse_value(args.value))
elif args.action == 'unset':
elif args.action == "set":
set_config_value(args.config_path, args.key_path, parse_value(args.value))
elif args.action == "unset":
unset_config_value(args.config_path, args.key_path)
elif args.action == 'add-item':
add_config_value(args.config_path, args.key_path,
parse_value(args.value))
elif args.action == 'remove-item':
remove_config_value(args.config_path, args.key_path,
parse_value(args.value))
elif args.action == 'reload':
elif args.action == "add-item":
add_config_value(args.config_path, args.key_path, parse_value(args.value))
elif args.action == "remove-item":
remove_config_value(args.config_path, args.key_path, parse_value(args.value))
elif args.action == "reload":
reload_component(args.component)
else:
argparser.print_help()
if __name__ == '__main__':
if __name__ == "__main__":
main()

175
tljh/configurer.py
View File

@@ -17,60 +17,51 @@ from .yaml import yaml
# Default configuration for tljh
# User provided config is merged into this
default = {
'base_url': '/',
'auth': {
'type': 'firstuseauthenticator.FirstUseAuthenticator',
'FirstUseAuthenticator': {
'create_users': False
}
"base_url": "/",
"auth": {
"type": "firstuseauthenticator.FirstUseAuthenticator",
"FirstUseAuthenticator": {"create_users": False},
},
'users': {
'allowed': [],
'banned': [],
'admin': [],
'extra_user_groups': {}
"users": {"allowed": [], "banned": [], "admin": [], "extra_user_groups": {}},
"limits": {
"memory": None,
"cpu": None,
},
'limits': {
'memory': None,
'cpu': None,
"http": {
"port": 80,
},
'http': {
'port': 80,
},
'https': {
'enabled': False,
'port': 443,
'tls': {
'cert': '',
'key': '',
"https": {
"enabled": False,
"port": 443,
"tls": {
"cert": "",
"key": "",
},
'letsencrypt': {
'email': '',
'domains': [],
"letsencrypt": {
"email": "",
"domains": [],
},
},
'traefik_api': {
'ip': "127.0.0.1",
'port': 8099,
'username': 'api_admin',
'password': '',
"traefik_api": {
"ip": "127.0.0.1",
"port": 8099,
"username": "api_admin",
"password": "",
},
'user_environment': {
'default_app': 'classic',
"user_environment": {
"default_app": "classic",
},
'services': {
'cull': {
'enabled': True,
'timeout': 600,
'every': 60,
'concurrency': 5,
'users': False,
'max_age': 0
"services": {
"cull": {
"enabled": True,
"timeout": 600,
"every": 60,
"concurrency": 5,
"users": False,
"max_age": 0,
},
'configurator': {
'enabled': False
}
}
"configurator": {"enabled": False},
},
}
@@ -118,14 +109,14 @@ def set_if_not_none(parent, key, value):
def load_traefik_api_credentials():
"""Load traefik api secret from a file"""
proxy_secret_path = os.path.join(STATE_DIR, 'traefik-api.secret')
proxy_secret_path = os.path.join(STATE_DIR, "traefik-api.secret")
if not os.path.exists(proxy_secret_path):
return {}
with open(proxy_secret_path) as f:
password = f.read()
return {
'traefik_api': {
'password': password,
"traefik_api": {
"password": password,
}
}
@@ -144,7 +135,7 @@ def update_base_url(c, config):
"""
Update base_url of JupyterHub through tljh config
"""
c.JupyterHub.base_url = config['base_url']
c.JupyterHub.base_url = config["base_url"]
def update_auth(c, config):
@@ -181,15 +172,17 @@ def update_auth(c, config):
c.JupyterHub.authenticator_class and any configured value being None won't
be set.
"""
tljh_auth_config = config['auth']
tljh_auth_config = config["auth"]
c.JupyterHub.authenticator_class = tljh_auth_config['type']
c.JupyterHub.authenticator_class = tljh_auth_config["type"]
for auth_key, auth_value in tljh_auth_config.items():
if not (auth_key[0] == auth_key[0].upper() and isinstance(auth_value, dict)):
if auth_key == 'type':
if auth_key == "type":
continue
raise ValueError(f"Error: auth.{auth_key} was ignored, it didn't look like a valid configuration")
raise ValueError(
f"Error: auth.{auth_key} was ignored, it didn't look like a valid configuration"
)
class_name = auth_key
class_config_to_set = auth_value
class_config = c[class_name]
@@ -201,77 +194,75 @@ def update_userlists(c, config):
"""
Set user whitelists & admin lists
"""
users = config['users']
users = config["users"]
c.Authenticator.allowed_users = set(users['allowed'])
c.Authenticator.blocked_users = set(users['banned'])
c.Authenticator.admin_users = set(users['admin'])
c.Authenticator.allowed_users = set(users["allowed"])
c.Authenticator.blocked_users = set(users["banned"])
c.Authenticator.admin_users = set(users["admin"])
def update_usergroups(c, config):
"""
Set user groups
"""
users = config['users']
c.UserCreatingSpawner.user_groups = users['extra_user_groups']
users = config["users"]
c.UserCreatingSpawner.user_groups = users["extra_user_groups"]
def update_limits(c, config):
"""
Set user server limits
"""
limits = config['limits']
limits = config["limits"]
c.Spawner.mem_limit = limits['memory']
c.Spawner.cpu_limit = limits['cpu']
c.Spawner.mem_limit = limits["memory"]
c.Spawner.cpu_limit = limits["cpu"]
def update_user_environment(c, config):
"""
Set user environment configuration
"""
user_env = config['user_environment']
user_env = config["user_environment"]
# Set default application users are launched into
if user_env['default_app'] == 'jupyterlab':
c.Spawner.default_url = '/lab'
elif user_env['default_app'] == 'nteract':
c.Spawner.default_url = '/nteract'
if user_env["default_app"] == "jupyterlab":
c.Spawner.default_url = "/lab"
elif user_env["default_app"] == "nteract":
c.Spawner.default_url = "/nteract"
def update_user_account_config(c, config):
c.SystemdSpawner.username_template = 'jupyter-{USERNAME}'
c.SystemdSpawner.username_template = "jupyter-{USERNAME}"
def update_traefik_api(c, config):
"""
Set traefik api endpoint credentials
"""
c.TraefikTomlProxy.traefik_api_username = config['traefik_api']['username']
c.TraefikTomlProxy.traefik_api_password = config['traefik_api']['password']
c.TraefikTomlProxy.traefik_api_username = config["traefik_api"]["username"]
c.TraefikTomlProxy.traefik_api_password = config["traefik_api"]["password"]
def set_cull_idle_service(config):
"""
Set Idle Culler service
"""
cull_cmd = [
sys.executable, '-m', 'jupyterhub_idle_culler'
]
cull_config = config['services']['cull']
cull_cmd = [sys.executable, "-m", "jupyterhub_idle_culler"]
cull_config = config["services"]["cull"]
print()
cull_cmd += ['--timeout=%d' % cull_config['timeout']]
cull_cmd += ['--cull-every=%d' % cull_config['every']]
cull_cmd += ['--concurrency=%d' % cull_config['concurrency']]
cull_cmd += ['--max-age=%d' % cull_config['max_age']]
if cull_config['users']:
cull_cmd += ['--cull-users']
cull_cmd += ["--timeout=%d" % cull_config["timeout"]]
cull_cmd += ["--cull-every=%d" % cull_config["every"]]
cull_cmd += ["--concurrency=%d" % cull_config["concurrency"]]
cull_cmd += ["--max-age=%d" % cull_config["max_age"]]
if cull_config["users"]:
cull_cmd += ["--cull-users"]
cull_service = {
'name': 'cull-idle',
'admin': True,
'command': cull_cmd,
"name": "cull-idle",
"admin": True,
"command": cull_cmd,
}
return cull_service
@@ -283,13 +274,15 @@ def set_configurator(config):
"""
HERE = os.path.abspath(os.path.dirname(__file__))
configurator_cmd = [
sys.executable, "-m", "jupyterhub_configurator.app",
f"--Configurator.config_file={HERE}/jupyterhub_configurator_config.py"
sys.executable,
"-m",
"jupyterhub_configurator.app",
f"--Configurator.config_file={HERE}/jupyterhub_configurator_config.py",
]
configurator_service = {
'name': 'configurator',
'url': 'http://127.0.0.1:10101',
'command': configurator_cmd,
"name": "configurator",
"url": "http://127.0.0.1:10101",
"command": configurator_cmd,
}
return configurator_service
@@ -298,9 +291,9 @@ def set_configurator(config):
def update_services(c, config):
c.JupyterHub.services = []
if config['services']['cull']['enabled']:
if config["services"]["cull"]["enabled"]:
c.JupyterHub.services.append(set_cull_idle_service(config))
if config['services']['configurator']['enabled']:
if config["services"]["configurator"]["enabled"]:
c.JupyterHub.services.append(set_configurator(config))
@@ -321,7 +314,7 @@ def _merge_dictionaries(a, b, path=None, update=True):
elif update:
a[key] = b[key]
else:
raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
raise Exception("Conflict at %s" % ".".join(path + [str(key)]))
else:
a[key] = b[key]
return a

10
tljh/hooks.py
View File

@@ -3,8 +3,8 @@ Hook specifications that pluggy plugins can override
"""
import pluggy
hookspec = pluggy.HookspecMarker('tljh')
hookimpl = pluggy.HookimplMarker('tljh')
hookspec = pluggy.HookspecMarker("tljh")
hookimpl = pluggy.HookimplMarker("tljh")
@hookspec
@@ -22,6 +22,7 @@ def tljh_extra_user_pip_packages():
"""
pass
@hookspec
def tljh_extra_hub_pip_packages():
"""
@@ -29,6 +30,7 @@ def tljh_extra_hub_pip_packages():
"""
pass
@hookspec
def tljh_extra_apt_packages():
"""
@@ -38,6 +40,7 @@ def tljh_extra_apt_packages():
"""
pass
@hookspec
def tljh_custom_jupyterhub_config(c):
"""
@@ -48,6 +51,7 @@ def tljh_custom_jupyterhub_config(c):
"""
pass
@hookspec
def tljh_config_post_install(config):
"""
@@ -60,6 +64,7 @@ def tljh_config_post_install(config):
"""
pass
@hookspec
def tljh_post_install():
"""
@@ -70,6 +75,7 @@ def tljh_post_install():
"""
pass
@hookspec
def tljh_new_user_create(username):
"""

196
tljh/installer.py
View File

@@ -46,18 +46,18 @@ def remove_chp():
Ensure CHP is not running
"""
if os.path.exists("/etc/systemd/system/configurable-http-proxy.service"):
if systemd.check_service_active('configurable-http-proxy.service'):
if systemd.check_service_active("configurable-http-proxy.service"):
try:
systemd.stop_service('configurable-http-proxy.service')
systemd.stop_service("configurable-http-proxy.service")
except subprocess.CalledProcessError:
logger.info("Cannot stop configurable-http-proxy...")
if systemd.check_service_enabled('configurable-http-proxy.service'):
if systemd.check_service_enabled("configurable-http-proxy.service"):
try:
systemd.disable_service('configurable-http-proxy.service')
systemd.disable_service("configurable-http-proxy.service")
except subprocess.CalledProcessError:
logger.info("Cannot disable configurable-http-proxy...")
try:
systemd.uninstall_unit('configurable-http-proxy.service')
systemd.uninstall_unit("configurable-http-proxy.service")
except subprocess.CalledProcessError:
logger.info("Cannot uninstall configurable-http-proxy...")
@@ -70,38 +70,36 @@ def ensure_jupyterhub_service(prefix):
remove_chp()
systemd.reload_daemon()
with open(os.path.join(HERE, 'systemd-units', 'jupyterhub.service')) as f:
with open(os.path.join(HERE, "systemd-units", "jupyterhub.service")) as f:
hub_unit_template = f.read()
with open(os.path.join(HERE, 'systemd-units', 'traefik.service')) as f:
with open(os.path.join(HERE, "systemd-units", "traefik.service")) as f:
traefik_unit_template = f.read()
#Set up proxy / hub secret token if it is not already setup
proxy_secret_path = os.path.join(STATE_DIR, 'traefik-api.secret')
# Set up proxy / hub secret token if it is not already setup
proxy_secret_path = os.path.join(STATE_DIR, "traefik-api.secret")
if not os.path.exists(proxy_secret_path):
with open(proxy_secret_path, 'w') as f:
with open(proxy_secret_path, "w") as f:
f.write(secrets.token_hex(32))
traefik.ensure_traefik_config(STATE_DIR)
unit_params = dict(
python_interpreter_path=sys.executable,
jupyterhub_config_path=os.path.join(HERE, 'jupyterhub_config.py'),
jupyterhub_config_path=os.path.join(HERE, "jupyterhub_config.py"),
install_prefix=INSTALL_PREFIX,
)
systemd.install_unit('jupyterhub.service', hub_unit_template.format(**unit_params))
systemd.install_unit('traefik.service', traefik_unit_template.format(**unit_params))
systemd.install_unit("jupyterhub.service", hub_unit_template.format(**unit_params))
systemd.install_unit("traefik.service", traefik_unit_template.format(**unit_params))
systemd.reload_daemon()
# If JupyterHub is running, we want to restart it.
systemd.restart_service('jupyterhub')
systemd.restart_service('traefik')
systemd.restart_service("jupyterhub")
systemd.restart_service("traefik")
# Mark JupyterHub & traefik to start at boot time
systemd.enable_service('jupyterhub')
systemd.enable_service('traefik')
systemd.enable_service("jupyterhub")
systemd.enable_service("traefik")
def ensure_jupyterhub_package(prefix):
@@ -117,12 +115,8 @@ def ensure_jupyterhub_package(prefix):
# Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically
# pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289
# build-essential is also generally useful to everyone involved, and required for pycurl
apt.install_packages([
'libssl-dev',
'libcurl4-openssl-dev',
'build-essential'
])
conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True)
apt.install_packages(["libssl-dev", "libcurl4-openssl-dev", "build-essential"])
conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True)
conda.ensure_pip_packages(
prefix,
@@ -135,7 +129,7 @@ def ensure_jupyterhub_package(prefix):
"jupyterhub-tmpauthenticator==0.6.*",
"oauthenticator==14.*",
"jupyterhub-idle-culler==1.*",
"git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba"
"git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba",
],
upgrade=True,
)
@@ -146,17 +140,17 @@ def ensure_usergroups():
"""
Sets up user groups & sudo rules
"""
user.ensure_group('jupyterhub-admins')
user.ensure_group('jupyterhub-users')
user.ensure_group("jupyterhub-admins")
user.ensure_group("jupyterhub-users")
logger.info("Granting passwordless sudo to JupyterHub admins...")
with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f:
with open("/etc/sudoers.d/jupyterhub-admins", "w") as f:
# JupyterHub admins should have full passwordless sudo access
f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n')
f.write("%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n")
# `sudo -E` should preserve the $PATH we set. This allows
# admins in jupyter terminals to do `sudo -E pip install <package>`,
# `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env.
f.write('Defaults exempt_group = jupyterhub-admins\n')
f.write("Defaults exempt_group = jupyterhub-admins\n")
def ensure_user_environment(user_requirements_txt_file):
@@ -165,50 +159,58 @@ def ensure_user_environment(user_requirements_txt_file):
"""
logger.info("Setting up user environment...")
miniconda_old_version = '4.5.4'
miniconda_new_version = '4.7.10'
miniconda_old_version = "4.5.4"
miniconda_new_version = "4.7.10"
# Install mambaforge using an installer from
# https://github.com/conda-forge/miniforge/releases
mambaforge_new_version = '4.10.3-7'
mambaforge_new_version = "4.10.3-7"
# Check system architecture, set appropriate installer checksum
if os.uname().machine == 'aarch64':
installer_sha256 = "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2"
elif os.uname().machine == 'x86_64':
installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474"
if os.uname().machine == "aarch64":
installer_sha256 = (
"ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2"
)
elif os.uname().machine == "x86_64":
installer_sha256 = (
"fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474"
)
# Check OS, set appropriate string for conda installer path
if os.uname().sysname != 'Linux':
if os.uname().sysname != "Linux":
raise OSError("TLJH is only supported on Linux platforms.")
# Then run `mamba --version` to get the conda and mamba versions
# Keep these in sync with tests/test_conda.py::prefix
mambaforge_conda_new_version = '4.10.3'
mambaforge_mamba_version = '0.16.0'
mambaforge_conda_new_version = "4.10.3"
mambaforge_mamba_version = "0.16.0"
if conda.check_miniconda_version(USER_ENV_PREFIX, mambaforge_conda_new_version):
conda_version = '4.10.3'
conda_version = "4.10.3"
elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version):
conda_version = '4.8.1'
conda_version = "4.8.1"
elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version):
conda_version = '4.5.8'
conda_version = "4.5.8"
# If no prior miniconda installation is found, we can install a newer version
else:
logger.info('Downloading & setting up user environment...')
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(v=mambaforge_new_version, arch=os.uname().machine)
with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path:
logger.info("Downloading & setting up user environment...")
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(
v=mambaforge_new_version, arch=os.uname().machine
)
with conda.download_miniconda_installer(
installer_url, installer_sha256
) as installer_path:
conda.install_miniconda(installer_path, USER_ENV_PREFIX)
conda_version = '4.10.3'
conda_version = "4.10.3"
conda.ensure_conda_packages(
USER_ENV_PREFIX,
[
# Conda's latest version is on conda much more so than on PyPI.
'conda==' + conda_version,
'mamba==' + mambaforge_mamba_version,
"conda==" + conda_version,
"mamba==" + mambaforge_mamba_version,
],
)
conda.ensure_pip_requirements(
USER_ENV_PREFIX,
os.path.join(HERE, 'requirements-base.txt'),
os.path.join(HERE, "requirements-base.txt"),
upgrade=True,
)
@@ -237,25 +239,25 @@ def ensure_admins(admin_password_list):
else:
config = {}
config['users'] = config.get('users', {})
config["users"] = config.get("users", {})
db_passw = os.path.join(STATE_DIR, 'passwords.dbm')
db_passw = os.path.join(STATE_DIR, "passwords.dbm")
admins = []
for admin_password_entry in admin_password_list:
for admin_password_pair in admin_password_entry:
if ":" in admin_password_pair:
admin, password = admin_password_pair.split(':')
admin, password = admin_password_pair.split(":")
admins.append(admin)
# Add admin:password to the db
password = bcrypt.hashpw(password.encode(), bcrypt.gensalt())
with dbm.open(db_passw, 'c', 0o600) as db:
with dbm.open(db_passw, "c", 0o600) as db:
db[admin] = password
else:
admins.append(admin_password_pair)
config['users']['admin'] = admins
config["users"]["admin"] = admins
with open(config_path, 'w+') as f:
with open(config_path, "w+") as f:
yaml.dump(config, f)
@@ -268,12 +270,12 @@ def ensure_jupyterhub_running(times=20):
for i in range(times):
try:
logger.info(f'Waiting for JupyterHub to come up ({i + 1}/{times} tries)')
logger.info(f"Waiting for JupyterHub to come up ({i + 1}/{times} tries)")
# Because we don't care at this level that SSL is valid, we can suppress
# InsecureRequestWarning for this request.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=InsecureRequestWarning)
requests.get('http://127.0.0.1', verify=False)
requests.get("http://127.0.0.1", verify=False)
return
except requests.HTTPError as h:
if h.response.status_code in [404, 502, 503]:
@@ -283,9 +285,9 @@ def ensure_jupyterhub_running(times=20):
# Everything else should immediately abort
raise
except requests.ConnectionError:
# Hub isn't up yet, sleep & loop
time.sleep(1)
continue
# Hub isn't up yet, sleep & loop
time.sleep(1)
continue
except Exception:
# Everything else should immediately abort
raise
@@ -305,14 +307,16 @@ def ensure_symlinks(prefix):
around this with sudo -E and extra entries in the sudoers file, but this
is far more secure at the cost of upsetting some FHS purists.
"""
tljh_config_src = os.path.join(prefix, 'bin', 'tljh-config')
tljh_config_dest = '/usr/bin/tljh-config'
tljh_config_src = os.path.join(prefix, "bin", "tljh-config")
tljh_config_dest = "/usr/bin/tljh-config"
if os.path.exists(tljh_config_dest):
if os.path.realpath(tljh_config_dest) != tljh_config_src:
# tljh-config exists that isn't ours. We should *not* delete this file,
# instead we throw an error and abort. Deleting files owned by other people
# while running as root is dangerous, especially with symlinks involved.
raise FileExistsError(f'/usr/bin/tljh-config exists but is not a symlink to {tljh_config_src}')
raise FileExistsError(
f"/usr/bin/tljh-config exists but is not a symlink to {tljh_config_src}"
)
else:
# We have a working symlink, so do nothing
return
@@ -328,9 +332,9 @@ def setup_plugins(plugins=None):
conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins, upgrade=True)
# Set up plugin infrastructure
pm = pluggy.PluginManager('tljh')
pm = pluggy.PluginManager("tljh")
pm.add_hookspecs(hooks)
pm.load_setuptools_entrypoints('tljh')
pm.load_setuptools_entrypoints("tljh")
return pm
@@ -343,17 +347,21 @@ def run_plugin_actions(plugin_manager):
# Install apt packages
apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages())))
if apt_packages:
logger.info('Installing {} apt packages collected from plugins: {}'.format(
len(apt_packages), ' '.join(apt_packages)
))
logger.info(
"Installing {} apt packages collected from plugins: {}".format(
len(apt_packages), " ".join(apt_packages)
)
)
apt.install_packages(apt_packages)
# Install hub pip packages
hub_pip_packages = list(set(itertools.chain(*hook.tljh_extra_hub_pip_packages())))
if hub_pip_packages:
logger.info('Installing {} hub pip packages collected from plugins: {}'.format(
len(hub_pip_packages), ' '.join(hub_pip_packages)
))
logger.info(
"Installing {} hub pip packages collected from plugins: {}".format(
len(hub_pip_packages), " ".join(hub_pip_packages)
)
)
conda.ensure_pip_packages(
HUB_ENV_PREFIX,
hub_pip_packages,
@@ -363,17 +371,21 @@ def run_plugin_actions(plugin_manager):
# Install conda packages
conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages())))
if conda_packages:
logger.info('Installing {} user conda packages collected from plugins: {}'.format(
len(conda_packages), ' '.join(conda_packages)
))
logger.info(
"Installing {} user conda packages collected from plugins: {}".format(
len(conda_packages), " ".join(conda_packages)
)
)
conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages)
# Install pip packages
user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages())))
if user_pip_packages:
logger.info('Installing {} user pip packages collected from plugins: {}'.format(
len(user_pip_packages), ' '.join(user_pip_packages)
))
logger.info(
"Installing {} user pip packages collected from plugins: {}".format(
len(user_pip_packages), " ".join(user_pip_packages)
)
)
conda.ensure_pip_packages(
USER_ENV_PREFIX,
user_pip_packages,
@@ -389,7 +401,7 @@ def ensure_config_yaml(plugin_manager):
Ensure we have a config.yaml present
"""
# ensure config dir exists and is private
for path in [CONFIG_DIR, os.path.join(CONFIG_DIR, 'jupyterhub_config.d')]:
for path in [CONFIG_DIR, os.path.join(CONFIG_DIR, "jupyterhub_config.d")]:
os.makedirs(path, mode=0o700, exist_ok=True)
migrator.migrate_config_files()
@@ -403,34 +415,28 @@ def ensure_config_yaml(plugin_manager):
hook = plugin_manager.hook
hook.tljh_config_post_install(config=config)
with open(CONFIG_FILE, 'w+') as f:
with open(CONFIG_FILE, "w+") as f:
yaml.dump(config, f)
def main():
from .log import init_logging
init_logging()
argparser = argparse.ArgumentParser()
argparser.add_argument(
'--admin',
nargs='*',
action='append',
help='List of usernames set to be admin'
"--admin", nargs="*", action="append", help="List of usernames set to be admin"
)
argparser.add_argument(
'--user-requirements-txt-url',
help='URL to a requirements.txt file that should be installed in the user environment'
"--user-requirements-txt-url",
help="URL to a requirements.txt file that should be installed in the user environment",
)
argparser.add_argument("--plugin", nargs="*", help="Plugin pip-specs to install")
argparser.add_argument(
'--plugin',
nargs='*',
help='Plugin pip-specs to install'
)
argparser.add_argument(
'--progress-page-server-pid',
"--progress-page-server-pid",
type=int,
help='The pid of the progress page server'
help="The pid of the progress page server",
)
args = argparser.parse_args()
@@ -465,5 +471,5 @@ def main():
logger.info("Done!")
if __name__ == '__main__':
if __name__ == "__main__":
main()

10
tljh/jupyterhub_config.py
View File

@@ -21,14 +21,14 @@ c.JupyterHub.hub_port = 15001
c.TraefikTomlProxy.should_start = False
dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules', 'rules.toml')
dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, "state", "rules", "rules.toml")
c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path
c.JupyterHub.proxy_class = TraefikTomlProxy
c.SystemdSpawner.extra_paths = [os.path.join(USER_ENV_PREFIX, 'bin')]
c.SystemdSpawner.default_shell = '/bin/bash'
c.SystemdSpawner.extra_paths = [os.path.join(USER_ENV_PREFIX, "bin")]
c.SystemdSpawner.default_shell = "/bin/bash"
# Drop the '-singleuser' suffix present in the default template
c.SystemdSpawner.unit_name_template = 'jupyter-{USERNAME}'
c.SystemdSpawner.unit_name_template = "jupyter-{USERNAME}"
tljh_config = configurer.load_config()
configurer.apply_config(tljh_config, c)
@@ -41,6 +41,6 @@ pm.hook.tljh_custom_jupyterhub_config(c=c)
# Load arbitrary .py config files if they exist.
# This is our escape hatch
extra_configs = sorted(glob(os.path.join(CONFIG_DIR, 'jupyterhub_config.d', '*.py')))
extra_configs = sorted(glob(os.path.join(CONFIG_DIR, "jupyterhub_config.d", "*.py")))
for ec in extra_configs:
load_subconfig(ec)

7
tljh/normalize.py
View File

@@ -17,8 +17,7 @@ def generate_system_username(username):
if len(username) < 26:
return username
userhash = hashlib.sha256(username.encode('utf-8')).hexdigest()
return '{username_trunc}-{hash}'.format(
username_trunc=username[:26],
hash=userhash[:5]
userhash = hashlib.sha256(username.encode("utf-8")).hexdigest()
return "{username_trunc}-{hash}".format(
username_trunc=username[:26], hash=userhash[:5]
)

59
tljh/systemd.py
View File

@@ -13,61 +13,43 @@ def reload_daemon():
Makes systemd discover new units.
"""
subprocess.run([
'systemctl',
'daemon-reload'
], check=True)
subprocess.run(["systemctl", "daemon-reload"], check=True)
def install_unit(name, unit, path='/etc/systemd/system'):
def install_unit(name, unit, path="/etc/systemd/system"):
"""
Install unit with given name
"""
with open(os.path.join(path, name), 'w') as f:
with open(os.path.join(path, name), "w") as f:
f.write(unit)
def uninstall_unit(name, path='/etc/systemd/system'):
def uninstall_unit(name, path="/etc/systemd/system"):
"""
Uninstall unit with given name
"""
subprocess.run([
'rm',
os.path.join(path, name)
], check=True)
subprocess.run(["rm", os.path.join(path, name)], check=True)
def start_service(name):
"""
Start service with given name.
"""
subprocess.run([
'systemctl',
'start',
name
], check=True)
subprocess.run(["systemctl", "start", name], check=True)
def stop_service(name):
"""
Start service with given name.
"""
subprocess.run([
'systemctl',
'stop',
name
], check=True)
subprocess.run(["systemctl", "stop", name], check=True)
def restart_service(name):
"""
Restart service with given name.
"""
subprocess.run([
'systemctl',
'restart',
name
], check=True)
subprocess.run(["systemctl", "restart", name], check=True)
def enable_service(name):
@@ -76,11 +58,7 @@ def enable_service(name):
This most likely makes the service start on bootup
"""
subprocess.run([
'systemctl',
'enable',
name
], check=True)
subprocess.run(["systemctl", "enable", name], check=True)
def disable_service(name):
@@ -89,11 +67,7 @@ def disable_service(name):
This most likely makes the service start on bootup
"""
subprocess.run([
'systemctl',
'disable',
name
], check=True)
subprocess.run(["systemctl", "disable", name], check=True)
def check_service_active(name):
@@ -101,25 +75,18 @@ def check_service_active(name):
Check if a service is currently active (running)
"""
try:
subprocess.run([
'systemctl',
'is-active',
name
], check=True)
subprocess.run(["systemctl", "is-active", name], check=True)
return True
except subprocess.CalledProcessError:
return False
def check_service_enabled(name):
"""
Check if a service is enabled
"""
try:
subprocess.run([
'systemctl',
'is-enabled',
name
], check=True)
subprocess.run(["systemctl", "is-enabled", name], check=True)
return True
except subprocess.CalledProcessError:
return False

27
tljh/traefik.py
View File

@@ -15,9 +15,9 @@ from tljh.configurer import load_config, _merge_dictionaries
# traefik 2.7.x is not supported yet, use v1.7.x for now
# see: https://github.com/jupyterhub/traefik-proxy/issues/97
machine = os.uname().machine
if machine == 'aarch64':
if machine == "aarch64":
plat = "linux-arm64"
elif machine == 'x86_64':
elif machine == "x86_64":
plat = "linux-amd64"
else:
raise OSError(f"Error. Platform: {os.uname().sysname} / {machine} Not supported.")
@@ -26,9 +26,10 @@ traefik_version = "1.7.33"
# record sha256 hashes for supported platforms here
checksums = {
"linux-amd64": "314ffeaa4cd8ed6ab7b779e9b6773987819f79b23c28d7ab60ace4d3683c5935",
"linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1"
"linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1",
}
def checksum_file(path):
"""Compute the sha256 checksum of a path"""
hasher = hashlib.sha256()
@@ -37,16 +38,13 @@ def checksum_file(path):
hasher.update(chunk)
return hasher.hexdigest()
def fatal_error(e):
# Retry only when connection is reset or we think we didn't download entire file
return str(e) != "ContentTooShort" and not isinstance(e, ConnectionResetError)
@backoff.on_exception(
backoff.expo,
Exception,
max_tries=2,
giveup=fatal_error
)
@backoff.on_exception(backoff.expo, Exception, max_tries=2, giveup=fatal_error)
def ensure_traefik_binary(prefix):
"""Download and install the traefik binary to a location identified by a prefix path such as '/opt/tljh/hub/'"""
traefik_bin = os.path.join(prefix, "bin", "traefik")
@@ -71,7 +69,7 @@ def ensure_traefik_binary(prefix):
response = requests.get(traefik_url)
if response.status_code == 206:
raise Exception("ContentTooShort")
with open(traefik_bin, 'wb') as f:
with open(traefik_bin, "wb") as f:
f.write(response.content)
os.chmod(traefik_bin, 0o755)
@@ -91,7 +89,7 @@ def compute_basic_auth(username, password):
def load_extra_config(extra_config_dir):
extra_configs = sorted(glob(os.path.join(extra_config_dir, '*.toml')))
extra_configs = sorted(glob(os.path.join(extra_config_dir, "*.toml")))
# Load the toml list of files into dicts and merge them
config = toml.load(extra_configs)
return config
@@ -104,9 +102,9 @@ def ensure_traefik_config(state_dir):
traefik_dynamic_config_dir = os.path.join(state_dir, "rules")
config = load_config()
config['traefik_api']['basic_auth'] = compute_basic_auth(
config['traefik_api']['username'],
config['traefik_api']['password'],
config["traefik_api"]["basic_auth"] = compute_basic_auth(
config["traefik_api"]["username"],
config["traefik_api"]["password"],
)
with open(os.path.join(os.path.dirname(__file__), "traefik.toml.tpl")) as f:
@@ -150,4 +148,3 @@ def ensure_traefik_config(state_dir):
# ensure acme.json exists and is private
with open(os.path.join(state_dir, "acme.json"), "a") as f:
os.fchmod(f.fileno(), 0o600)

44
tljh/user.py
View File

@@ -25,17 +25,9 @@ def ensure_user(username):
# User doesn't exist, time to create!
pass
subprocess.check_call([
'useradd',
'--create-home',
username
])
subprocess.check_call(["useradd", "--create-home", username])
subprocess.check_call([
'chmod',
'o-rwx',
expanduser(f'~{username}')
])
subprocess.check_call(["chmod", "o-rwx", expanduser(f"~{username}")])
pm = get_plugin_manager()
pm.hook.tljh_new_user_create(username=username)
@@ -51,22 +43,14 @@ def remove_user(username):
# User doesn't exist, nothing to do
return
subprocess.check_call([
'deluser',
'--quiet',
username
])
subprocess.check_call(["deluser", "--quiet", username])
def ensure_group(groupname):
"""
Ensure given group exists
"""
subprocess.check_call([
'groupadd',
'--force',
groupname
])
subprocess.check_call(["groupadd", "--force", groupname])
def remove_group(groupname):
@@ -79,11 +63,7 @@ def remove_group(groupname):
# Group doesn't exist, nothing to do
return
subprocess.check_call([
'delgroup',
'--quiet',
groupname
])
subprocess.check_call(["delgroup", "--quiet", groupname])
def ensure_user_group(username, groupname):
@@ -96,12 +76,7 @@ def ensure_user_group(username, groupname):
if username in group.gr_mem:
return
subprocess.check_call([
'gpasswd',
'--add',
username,
groupname
])
subprocess.check_call(["gpasswd", "--add", username, groupname])
def remove_user_group(username, groupname):
@@ -112,9 +87,4 @@ def remove_user_group(username, groupname):
if username not in group.gr_mem:
return
subprocess.check_call([
'gpasswd',
'--delete',
username,
groupname
])
subprocess.check_call(["gpasswd", "--delete", username, groupname])

19
tljh/user_creating_spawner.py
View File

@@ -5,12 +5,14 @@ from systemdspawner import SystemdSpawner
from traitlets import Dict, Unicode, List
from jupyterhub_configurator.mixins import ConfiguratorSpawnerMixin
class CustomSpawner(SystemdSpawner):
"""
SystemdSpawner with user creation on spawn.
FIXME: Remove this somehow?
"""
user_groups = Dict(key_trait=Unicode(), value_trait=List(Unicode()), config=True)
def start(self):
@@ -18,28 +20,31 @@ class CustomSpawner(SystemdSpawner):
Perform system user activities before starting server
"""
# FIXME: Move this elsewhere? Into the Authenticator?
system_username = generate_system_username('jupyter-' + self.user.name)
system_username = generate_system_username("jupyter-" + self.user.name)
# FIXME: This is a hack. Allow setting username directly instead
self.username_template = system_username
user.ensure_user(system_username)
user.ensure_user_group(system_username, 'jupyterhub-users')
user.ensure_user_group(system_username, "jupyterhub-users")
if self.user.admin:
user.ensure_user_group(system_username, 'jupyterhub-admins')
user.ensure_user_group(system_username, "jupyterhub-admins")
else:
user.remove_user_group(system_username, 'jupyterhub-admins')
user.remove_user_group(system_username, "jupyterhub-admins")
if self.user_groups:
for group, users in self.user_groups.items():
if self.user.name in users:
user.ensure_user_group(system_username, group)
return super().start()
cfg = configurer.load_config()
# Use the jupyterhub-configurator mixin only if configurator is enabled
# otherwise, any bugs in the configurator backend will stop new user spawns!
if cfg['services']['configurator']['enabled']:
if cfg["services"]["configurator"]["enabled"]:
# Dynamically create the Spawner class using `type`(https://docs.python.org/3/library/functions.html?#type),
# based on whether or not it should inherit from ConfiguratorSpawnerMixin
UserCreatingSpawner = type('UserCreatingSpawner', (ConfiguratorSpawnerMixin, CustomSpawner), {})
UserCreatingSpawner = type(
"UserCreatingSpawner", (ConfiguratorSpawnerMixin, CustomSpawner), {}
)
else:
UserCreatingSpawner = type('UserCreatingSpawner', (CustomSpawner,), {})
UserCreatingSpawner = type("UserCreatingSpawner", (CustomSpawner,), {})

28
tljh/utils.py
View File

@@ -23,21 +23,27 @@ def run_subprocess(cmd, *args, **kwargs):
In TLJH, this sends successful output to the installer log,
and failed output directly to the user's screen
"""
logger = logging.getLogger('tljh')
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs)
printable_command = ' '.join(cmd)
logger = logging.getLogger("tljh")
proc = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs
)
printable_command = " ".join(cmd)
if proc.returncode != 0:
# Our process failed! Show output to the user
logger.error('Ran {command} with exit code {code}'.format(
command=printable_command, code=proc.returncode
))
logger.error(
"Ran {command} with exit code {code}".format(
command=printable_command, code=proc.returncode
)
)
logger.error(proc.stdout.decode())
raise subprocess.CalledProcessError(cmd=cmd, returncode=proc.returncode)
else:
# This goes into installer.log
logger.debug('Ran {command} with exit code {code}'.format(
command=printable_command, code=proc.returncode
))
logger.debug(
"Ran {command} with exit code {code}".format(
command=printable_command, code=proc.returncode
)
)
# This produces multi line log output, unfortunately. Not sure how to fix.
# For now, prioritizing human readability over machine readability.
logger.debug(proc.stdout.decode())
@@ -48,8 +54,8 @@ def get_plugin_manager():
Return plugin manager instance
"""
# Set up plugin infrastructure
pm = pluggy.PluginManager('tljh')
pm = pluggy.PluginManager("tljh")
pm.add_hookspecs(hooks)
pm.load_setuptools_entrypoints('tljh')
pm.load_setuptools_entrypoints("tljh")
return pm