zhangyiss/the-littlest-jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/the-littlest-jupyterhub.git synced 2025-12-18 21:54:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into conda-channels

Browse Source
This commit is contained in:
Erik Sundell
2024-09-04 13:16:58 +02:00
parent 291c096a17 50c1b5f894
commit 7a491e403c
42 changed files with 351 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tljh/apt.py
View File

@@ -1,6 +1,7 @@
"""
Utilities for working with the apt package manager
"""
import os
import subprocess

1
tljh/conda.py
View File

@@ -1,6 +1,7 @@
"""
Wrap conda commandline program
"""
import contextlib
import hashlib
import json

163
tljh/config.py
View File

@@ -154,92 +154,137 @@ def remove_item_from_config(config, property_path, value):
return config_copy
def validate_config(config, validate):
"""
Validate changes to the config with tljh-config against the schema
"""
import jsonschema
from .config_schema import config_schema
try:
jsonschema.validate(instance=config, schema=config_schema)
except jsonschema.exceptions.ValidationError as e:
if validate:
print(
f"Config validation error: {e.message}.\n"
"You can still apply this change without validation by re-running your command with the --no-validate flag.\n"
"If you think this validation error is incorrect, please report it to https://github.com/jupyterhub/the-littlest-jupyterhub/issues."
)
exit()
def show_config(config_path):
"""
Pretty print config from given config_path
"""
try:
with open(config_path) as f:
config = yaml.load(f)
except FileNotFoundError:
config = {}
config = get_current_config(config_path)
yaml.dump(config, sys.stdout)
def set_config_value(config_path, key_path, value):
def set_config_value(config_path, key_path, value, validate=True):
"""
Set key at key_path in config_path to value
"""
# FIXME: Have a file lock here
# FIXME: Validate schema here
from filelock import FileLock, Timeout
lock_file = f"{config_path}.lock"
lock = FileLock(lock_file)
try:
with open(config_path) as f:
config = yaml.load(f)
except FileNotFoundError:
config = {}
with lock.acquire(timeout=1):
config = get_current_config(config_path)
config = set_item_in_config(config, key_path, value)
validate_config(config, validate)
config = set_item_in_config(config, key_path, value)
with open(config_path, "w") as f:
yaml.dump(config, f)
with open(config_path, "w") as f:
yaml.dump(config, f)
except Timeout:
print(f"Another instance of tljh-config holds the lock {lock_file}")
exit(1)
def unset_config_value(config_path, key_path):
def unset_config_value(config_path, key_path, validate=True):
"""
Unset key at key_path in config_path
"""
# FIXME: Have a file lock here
# FIXME: Validate schema here
from filelock import FileLock, Timeout
lock_file = f"{config_path}.lock"
lock = FileLock(lock_file)
try:
with open(config_path) as f:
config = yaml.load(f)
except FileNotFoundError:
config = {}
with lock.acquire(timeout=1):
config = get_current_config(config_path)
config = unset_item_from_config(config, key_path)
validate_config(config, validate)
config = unset_item_from_config(config, key_path)
with open(config_path, "w") as f:
yaml.dump(config, f)
with open(config_path, "w") as f:
yaml.dump(config, f)
except Timeout:
print(f"Another instance of tljh-config holds the lock {lock_file}")
exit(1)
def add_config_value(config_path, key_path, value):
def add_config_value(config_path, key_path, value, validate=True):
"""
Add value to list at key_path
"""
# FIXME: Have a file lock here
# FIXME: Validate schema here
from filelock import FileLock, Timeout
lock_file = f"{config_path}.lock"
lock = FileLock(lock_file)
try:
with open(config_path) as f:
config = yaml.load(f)
except FileNotFoundError:
config = {}
with lock.acquire(timeout=1):
config = get_current_config(config_path)
config = add_item_to_config(config, key_path, value)
validate_config(config, validate)
config = add_item_to_config(config, key_path, value)
with open(config_path, "w") as f:
yaml.dump(config, f)
with open(config_path, "w") as f:
yaml.dump(config, f)
except Timeout:
print(f"Another instance of tljh-config holds the lock {lock_file}")
exit(1)
def remove_config_value(config_path, key_path, value):
def remove_config_value(config_path, key_path, value, validate=True):
"""
Remove value from list at key_path
"""
# FIXME: Have a file lock here
# FIXME: Validate schema here
from filelock import FileLock, Timeout
lock_file = f"{config_path}.lock"
lock = FileLock(lock_file)
try:
with lock.acquire(timeout=1):
config = get_current_config(config_path)
config = remove_item_from_config(config, key_path, value)
validate_config(config, validate)
with open(config_path, "w") as f:
yaml.dump(config, f)
except Timeout:
print(f"Another instance of tljh-config holds the lock {lock_file}")
exit(1)
def get_current_config(config_path):
"""
Retrieve the current config at config_path
"""
try:
with open(config_path) as f:
config = yaml.load(f)
return yaml.load(f)
except FileNotFoundError:
config = {}
config = remove_item_from_config(config, key_path, value)
with open(config_path, "w") as f:
yaml.dump(config, f)
return {}
def check_hub_ready():
"""
Checks that hub is running.
"""
from .configurer import load_config
base_url = load_config()["base_url"]
@@ -336,6 +381,18 @@ def main(argv=None):
argparser.add_argument(
"--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file"
)
argparser.add_argument(
"--validate", action="store_true", help="Validate the TLJH config"
)
argparser.add_argument(
"--no-validate",
dest="validate",
action="store_false",
help="Do not validate the TLJH config",
)
argparser.set_defaults(validate=True)
subparsers = argparser.add_subparsers(dest="action")
show_parser = subparsers.add_parser("show", help="Show current configuration")
@@ -383,13 +440,19 @@ def main(argv=None):
if args.action == "show":
show_config(args.config_path)
elif args.action == "set":
set_config_value(args.config_path, args.key_path, parse_value(args.value))
set_config_value(
args.config_path, args.key_path, parse_value(args.value), args.validate
)
elif args.action == "unset":
unset_config_value(args.config_path, args.key_path)
unset_config_value(args.config_path, args.key_path, args.validate)
elif args.action == "add-item":
add_config_value(args.config_path, args.key_path, parse_value(args.value))
add_config_value(
args.config_path, args.key_path, parse_value(args.value), args.validate
)
elif args.action == "remove-item":
remove_config_value(args.config_path, args.key_path, parse_value(args.value))
remove_config_value(
args.config_path, args.key_path, parse_value(args.value), args.validate
)
elif args.action == "reload":
reload_component(args.component)
else:

117
tljh/config_schema.py Normal file
View File

@@ -0,0 +1,117 @@
"""
The schema against which the TLJH config file can be validated.
Validation occurs when changing values with tljh-config.
"""
config_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Littlest JupyterHub YAML config file",
"definitions": {
"BaseURL": {
"type": "string",
},
"Users": {
"type": "object",
"additionalProperties": False,
"properties": {
"extra_user_groups": {"type": "object", "items": {"type": "string"}},
"allowed": {"type": "array", "items": {"type": "string"}},
"banned": {"type": "array", "items": {"type": "string"}},
"admin": {"type": "array", "items": {"type": "string"}},
},
},
"Services": {
"type": "object",
"properties": {
"cull": {
"type": "object",
"additionalProperties": False,
"properties": {
"enabled": {"type": "boolean"},
"timeout": {"type": "integer"},
"every": {"type": "integer"},
"concurrency": {"type": "integer"},
"users": {"type": "boolean"},
"max_age": {"type": "integer"},
"remove_named_servers": {"type": "boolean"},
},
}
},
},
"HTTP": {
"type": "object",
"additionalProperties": False,
"properties": {
"address": {"type": "string", "format": "ipv4"},
"port": {"type": "integer"},
},
},
"HTTPS": {
"type": "object",
"additionalProperties": False,
"properties": {
"enabled": {"type": "boolean"},
"address": {"type": "string", "format": "ipv4"},
"port": {"type": "integer"},
"tls": {"$ref": "#/definitions/TLS"},
"letsencrypt": {"$ref": "#/definitions/LetsEncrypt"},
},
},
"LetsEncrypt": {
"type": "object",
"additionalProperties": False,
"properties": {
"email": {"type": "string", "format": "email"},
"domains": {
"type": "array",
"items": {"type": "string", "format": "hostname"},
},
"staging": {"type": "boolean"},
},
},
"TLS": {
"type": "object",
"additionalProperties": False,
"properties": {"key": {"type": "string"}, "cert": {"type": "string"}},
},
"Limits": {
"description": "User CPU and memory limits.",
"type": "object",
"additionalProperties": False,
"properties": {"memory": {"type": "string"}, "cpu": {"type": "integer"}},
},
"UserEnvironment": {
"type": "object",
"additionalProperties": False,
"properties": {
"default_app": {
"type": "string",
"enum": ["jupyterlab", "classic"],
"default": "jupyterlab",
}
},
},
"TraefikAPI": {
"type": "object",
"additionalProperties": False,
"properties": {
"ip": {"type": "string", "format": "ipv4"},
"port": {"type": "integer"},
"username": {"type": "string"},
"password": {"type": "string"},
},
},
},
"properties": {
"additionalProperties": False,
"base_url": {"$ref": "#/definitions/BaseURL"},
"user_environment": {"$ref": "#/definitions/UserEnvironment"},
"users": {"$ref": "#/definitions/Users"},
"limits": {"$ref": "#/definitions/Limits"},
"https": {"$ref": "#/definitions/HTTPS"},
"http": {"$ref": "#/definitions/HTTP"},
"traefik_api": {"$ref": "#/definitions/TraefikAPI"},
"services": {"$ref": "#/definitions/Services"},
},
}

8
tljh/configurer.py
View File

@@ -199,6 +199,14 @@ def update_userlists(c, config):
"""
users = config["users"]
if (
not users["allowed"]
and config["auth"]["type"] == default["auth"]["type"]
and "allow_all" not in c.FirstUseAuthenticator
):
# _default_ authenticator, enable allow_all if no users specified
c.FirstUseAuthenticator.allow_all = True
c.Authenticator.allowed_users = set(users["allowed"])
c.Authenticator.blocked_users = set(users["banned"])
c.Authenticator.admin_users = set(users["admin"])

1
tljh/hooks.py
View File

@@ -1,6 +1,7 @@
"""
Hook specifications that pluggy plugins can override
"""
import pluggy
hookspec = pluggy.HookspecMarker("tljh")

31
tljh/installer.py
View File

@@ -136,13 +136,13 @@ def ensure_usergroups():
f.write("Defaults exempt_group = jupyterhub-admins\n")
# Install mambaforge using an installer from
# Install miniforge using an installer from
# https://github.com/conda-forge/miniforge/releases
MAMBAFORGE_VERSION = "23.1.0-1"
MINIFORGE_VERSION = "24.7.1-0"
# sha256 checksums
MAMBAFORGE_CHECKSUMS = {
"aarch64": "d9d89c9e349369702171008d9ee7c5ce80ed420e5af60bd150a3db4bf674443a",
"x86_64": "cfb16c47dc2d115c8b114280aa605e322173f029fdb847a45348bf4bd23c62ab",
MINIFORGE_CHECKSUMS = {
"aarch64": "7a3372268b45679584043b4ba1e0318ee5027384a8d330f2d991b14d815d6a6d",
"x86_64": "b64f77042cf8eafd31ced64f9253a74fb85db63545fe167ba5756aea0e8125be",
}
# minimum versions of packages
@@ -156,22 +156,22 @@ MINIMUM_VERSIONS = {
}
def _mambaforge_url(version=MAMBAFORGE_VERSION, arch=None):
"""Return (URL, checksum) for mambaforge download for a given version and arch
def _miniforge_url(version=MINIFORGE_VERSION, arch=None):
"""Return (URL, checksum) for miniforge download for a given version and arch
Default values provided for both version and arch
"""
if arch is None:
arch = os.uname().machine
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Miniforge3-{v}-Linux-{arch}.sh".format(
v=version,
arch=arch,
)
# Check system architecture, set appropriate installer checksum
checksum = MAMBAFORGE_CHECKSUMS.get(arch)
checksum = MINIFORGE_CHECKSUMS.get(arch)
if not checksum:
raise ValueError(
f"Unsupported architecture: {arch}. TLJH only supports {','.join(MAMBAFORGE_CHECKSUMS.keys())}"
f"Unsupported architecture: {arch}. TLJH only supports {','.join(MINIFORGE_CHECKSUMS.keys())}"
)
return installer_url, checksum
@@ -198,7 +198,7 @@ def ensure_user_environment(user_requirements_txt_file):
raise OSError(msg)
logger.info("Downloading & setting up user environment...")
installer_url, installer_sha256 = _mambaforge_url()
installer_url, installer_sha256 = _miniforge_url()
with conda.download_miniconda_installer(
installer_url, installer_sha256
) as installer_path:
@@ -242,11 +242,10 @@ def ensure_user_environment(user_requirements_txt_file):
)
to_upgrade.append(pkg)
# force reinstall conda/mamba to ensure a basically consistent env
# avoids issues with RemoveError: 'requests' is a dependency of conda
# only do this for 'old' conda versions known to have a problem
# we don't know how old, but we know 4.10 is affected and 23.1 is not
if not is_fresh_install and V(package_versions.get("conda", "0")) < V("23.1"):
# force reinstall conda/mamba to ensure conda doesn't raise error
# "RemoveError: 'requests' is a dependency of conda" later on when
# conda/mamba is used to install/upgrade something
if not is_fresh_install:
# force-reinstall doesn't upgrade packages
# it reinstalls them in-place
# only reinstall packages already present

1
tljh/log.py
View File

@@ -1,4 +1,5 @@
"""Setup tljh logging"""
import logging
import os

1
tljh/normalize.py
View File

@@ -1,6 +1,7 @@
"""
Functions to normalize various inputs
"""
import hashlib

7
tljh/requirements-hub-env.txt
View File

@@ -8,13 +8,13 @@
# If a dependency is bumped to a new major version, we should make a major
# version release of tljh.
#
jupyterhub>=4.0.2,<5
jupyterhub>=5.1.0,<6
jupyterhub-systemdspawner>=1.0.1,<2
jupyterhub-firstuseauthenticator>=1.0.0,<2
jupyterhub-nativeauthenticator>=1.2.0,<2
jupyterhub-ldapauthenticator>=1.3.2,<2
jupyterhub-tmpauthenticator>=1.0.0,<2
oauthenticator>=16.0.4,<17
oauthenticator[azuread]>=16.0.4,<17
jupyterhub-idle-culler>=1.2.1,<2
# pycurl is installed to improve reliability and performance for when JupyterHub
@@ -26,3 +26,6 @@ jupyterhub-idle-culler>=1.2.1,<2
# ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289
#
pycurl>=7.45.2,<8
# filelock is used to help us do atomic operations on config file(s)
filelock>=3.15.4,<4

2
tljh/systemd-units/jupyterhub.service
View File

@@ -18,7 +18,7 @@ Environment=TLJH_INSTALL_PREFIX={install_prefix}
Environment=PATH={install_prefix}/hub/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# Run upgrade-db before starting, in case Hub version has changed
# This is a no-op when no db exists or no upgrades are needed
ExecStart={python_interpreter_path} -m jupyterhub.app -f {jupyterhub_config_path} --upgrade-db
ExecStart={python_interpreter_path} -m jupyterhub -f {jupyterhub_config_path} --upgrade-db
[Install]
# Start service when system boots

1
tljh/systemd.py
View File

@@ -3,6 +3,7 @@ Wraps systemctl to install, uninstall, start & stop systemd services.
If we use a debian package instead, we can get rid of all this code.
"""
import os
import subprocess

1
tljh/traefik.py
View File

@@ -1,4 +1,5 @@
"""Traefik installation and setup"""
import hashlib
import io
import logging

1
tljh/user.py
View File

@@ -3,6 +3,7 @@ User management for tljh.
Supports minimal user & group management
"""
import grp
import pwd
import subprocess

1
tljh/utils.py
View File

@@ -1,6 +1,7 @@
"""
Miscellaneous functions useful in at least two places unrelated to each other
"""
import logging
import re
import subprocess

1
tljh/yaml.py
View File

@@ -3,6 +3,7 @@
ensures the same yaml settings for reading/writing
throughout tljh
"""
from ruamel.yaml import YAML
from ruamel.yaml.composer import Composer