diff --git a/.github/integration-test.py b/.github/integration-test.py index fcb8768..73ef8ba 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -167,15 +167,23 @@ def run_test( command = f"python3 /srv/src/bootstrap/bootstrap.py --version={upgrade_from}" run_command(container_name, command) + # show user environment + command = "/opt/tljh/user/bin/mamba list" + run_command(container_name, command) + command = f"python3 /srv/src/bootstrap/bootstrap.py {' '.join(installer_args)}" run_command(container_name, command) + # show user environment (again if upgrade) + command = "/opt/tljh/user/bin/mamba list" + run_command(container_name, command) + # Install pkgs from requirements in hub's pip, where # the bootstrap script installed the others command = "/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt" run_command(container_name, command) - # show environment + # show hub environment command = "/opt/tljh/hub/bin/python3 -m pip freeze" run_command(container_name, command) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index feba78d..6e63a6f 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -36,12 +36,18 @@ jobs: - name: "Debian 11, Py 3.9" distro_image: "debian:11" extra_flags: "" + - name: "Debian 12, Py 3.11" + distro_image: "debian:12" + extra_flags: "" - name: "Ubuntu 20.04, Py 3.8" distro_image: "ubuntu:20.04" extra_flags: "" - name: "Ubuntu 22.04 Py 3.10" distro_image: "ubuntu:22.04" extra_flags: "" + - name: "Ubuntu 24.04 Py 3.12" + distro_image: "ubuntu:24.04" + extra_flags: "" - name: "Ubuntu 22.04, Py 3.10, from main" distro_image: "ubuntu:22.04" extra_flags: --upgrade-from=main @@ -53,8 +59,8 @@ jobs: extra_flags: --upgrade-from=0.2.0 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" @@ -116,8 +122,8 @@ jobs: distro_image: "ubuntu:22.04" steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index ea7e1e2..154bd1c 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -48,17 +48,18 @@ jobs: python_version: "3.10" steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python_version }}" - - name: Install venv, git and setup venv + - name: Install venv, git, pip and setup venv run: | export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install --yes \ python3-venv \ + python3-pip \ bzip2 \ git @@ -70,7 +71,7 @@ jobs: # completion. Make sure to update the key to bust the cache # properly if you make a change that should influence it. - name: Load cached Python dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /srv/venv/ key: >- @@ -93,4 +94,4 @@ jobs: run: pytest tests timeout-minutes: 15 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec7d213..688a10e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.17.0 hooks: - id: pyupgrade args: @@ -22,7 +22,7 @@ repos: # Autoformat: Python code - repo: https://github.com/PyCQA/autoflake - rev: v2.2.1 + rev: v2.3.1 hooks: - id: autoflake # args ref: https://github.com/PyCQA/autoflake#advanced-usage @@ -31,25 +31,25 @@ repos: # Autoformat: Python code - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 24.8.0 hooks: - id: black # Autoformat: markdown, yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier # Misc... - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available hooks: # Autoformat: Makes sure files end in a newline and only a newline. @@ -64,7 +64,7 @@ repos: # Lint: Python code - repo: https://github.com/pycqa/flake8 - rev: "6.1.0" + rev: "7.1.1" hooks: - id: flake8 diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index ac52039..f69aceb 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -42,6 +42,7 @@ Command line flags, from "bootstrap.py --help": can also pass a branch name such as 'main' or a commit hash. """ + import logging import multiprocessing import os diff --git a/dev-requirements.txt b/dev-requirements.txt index 672ad32..f3b24c9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ +filelock packaging pytest pytest-cov diff --git a/docs/howto/admin/https.md b/docs/howto/admin/https.md index 1cdb459..449ef41 100644 --- a/docs/howto/admin/https.md +++ b/docs/howto/admin/https.md @@ -89,6 +89,7 @@ If so, you can tell your deployment to use these files: sudo tljh-config set https.enabled true sudo tljh-config set https.tls.key /etc/mycerts/mydomain.key sudo tljh-config set https.tls.cert /etc/mycerts/mydomain.cert +sudo tljh-config add-item https.tls.domains yourhub.yourdomain.edu ``` Once you have loaded this, your config should look like: @@ -103,6 +104,8 @@ https: tls: key: /etc/mycerts/mydomain.key cert: /etc/mycerts/mydomain.cert + domains: + - yourhub.yourdomain.edu ``` Finally, you can reload the proxy to load the new configuration: diff --git a/docs/howto/admin/systemd.md b/docs/howto/admin/systemd.md index 468cea7..f1716d9 100644 --- a/docs/howto/admin/systemd.md +++ b/docs/howto/admin/systemd.md @@ -23,7 +23,7 @@ PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes Environment=TLJH_INSTALL_PREFIX=/opt/tljh -ExecStart=/opt/tljh/hub/bin/python3 -m jupyterhub.app -f jupyterhub_config.py --upgrade-db +ExecStart=/opt/tljh/hub/bin/python3 -m jupyterhub -f jupyterhub_config.py --upgrade-db [Install] WantedBy=multi-user.target diff --git a/docs/howto/auth/github.md b/docs/howto/auth/github.md index 47fed0a..2e5baf5 100644 --- a/docs/howto/auth/github.md +++ b/docs/howto/auth/github.md @@ -91,10 +91,16 @@ For more information on `tljh-config`, see [](/topic/tljh-config). 4. Tell your JupyterHub to _use_ the GitHub OAuthenticator for authentication: ``` - sudo tljh-config set auth.type oauthenticator.github.GitHubOAuthenticator + sudo tljh-config set auth.type github ``` -5. Restart your JupyterHub so that new users see these changes: +5. Tell JupyterHub which users to allow, if you haven't already: + + ``` + sudo tljh-config add-item users.allowed good-user_1 + ``` + +6. Restart your JupyterHub so that new users see these changes: ``` sudo tljh-config reload diff --git a/docs/howto/auth/nativeauth.md b/docs/howto/auth/nativeauth.md index 370e2e2..22a56f7 100644 --- a/docs/howto/auth/nativeauth.md +++ b/docs/howto/auth/nativeauth.md @@ -42,4 +42,4 @@ tljh-config reload ## Optional features -More optional features are available on the `authenticator documentation ` +More optional features are available on the [authenticator documentation](https://native-authenticator.readthedocs.io/en/latest/) diff --git a/docs/install/amazon.md b/docs/install/amazon.md index dd042be..75fdd4c 100644 --- a/docs/install/amazon.md +++ b/docs/install/amazon.md @@ -139,10 +139,10 @@ Let's create the server on which we can run JupyterHub. SSH to connect (port 22). If you have never used your Amazon account before, you'll have to select - **Create a new security group**. You should give it a disitnguishing name + **Create a new security group**. You should give it a distinctive name under **Security group name** - such as `ssh_web` for future reference. If you have, one from before you can - select it and adjust it to have the rules you need, if you prefer. + such as `ssh_web` for future reference. If you already have a security group, + you can select it and adjust it to have the rules you need, if you prefer. The rules will default to include `SSH`. Leave that there, and then click on the **Add Rule** button. Under **Type** for the new rule, change the field diff --git a/docs/topic/installer-actions.md b/docs/topic/installer-actions.md index 7dc26ab..b738bf2 100644 --- a/docs/topic/installer-actions.md +++ b/docs/topic/installer-actions.md @@ -26,7 +26,7 @@ sudo rm -rf /opt/tljh/hub ## User environment -By default, a `mambaforge` conda environment is installed in `/opt/tljh/user`. This contains +By default, a `miniforge` conda environment is installed in `/opt/tljh/user`. This contains the notebook interface used to launch all users, and the various packages available to all users. The environment is owned by the `root` user. JupyterHub admins may use to `sudo -E conda install` or `sudo -E pip install` packages into this environment. diff --git a/integration-tests/Dockerfile b/integration-tests/Dockerfile index c1c73d8..eb116eb 100644 --- a/integration-tests/Dockerfile +++ b/integration-tests/Dockerfile @@ -1,5 +1,5 @@ # Systemd inside a Docker container, for CI only -ARG BASE_IMAGE=ubuntu:20.04 +ARG BASE_IMAGE=ubuntu:22.04 FROM $BASE_IMAGE # DEBIAN_FRONTEND is set to avoid being asked for input and hang during build: @@ -29,8 +29,8 @@ RUN systemctl set-default multi-user.target STOPSIGNAL SIGRTMIN+3 # Uncomment these lines for a development install -#ENV TLJH_BOOTSTRAP_DEV=yes -#ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src -#ENV PATH=/opt/tljh/hub/bin:${PATH} +# ENV TLJH_BOOTSTRAP_DEV=yes +# ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src +# ENV PATH=/opt/tljh/hub/bin:${PATH} CMD ["/bin/bash", "-c", "exec /lib/systemd/systemd --log-target=journal 3>&1"] diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index 8b663b3..c3f978d 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -1,6 +1,7 @@ """ Simplest plugin that exercises all the hooks defined in tljh/hooks.py. """ + from tljh.hooks import hookimpl @@ -16,7 +17,7 @@ def tljh_extra_user_conda_channels(): @hookimpl def tljh_extra_user_pip_packages(): - return ["django"] + return ["simplejson"] @hookimpl diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index c086c7f..3ef31e4 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,3 +1,4 @@ +filelock pytest pytest-cov pytest-asyncio diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 09ae140..eac9c92 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -9,6 +9,7 @@ FIXME: The last test stands out and could be part of the other tests, and the first two could be more like unit tests. Ideally, this file is significantly reduced. """ + import concurrent.futures import os import subprocess diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 55a35cd..939175d 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -33,7 +33,7 @@ def test_hub_version(): r = requests.get(HUB_URL + "/hub/api") r.raise_for_status() info = r.json() - assert V("4") <= V(info["version"]) <= V("5") + assert V("5.1") <= V(info["version"]) <= V("6") async def test_user_code_execute(): @@ -59,9 +59,9 @@ async def test_user_code_execute(): async with User(username, HUB_URL, partial(login_dummy, password="")) as u: assert await u.login() - await u.ensure_server_simulate(timeout=60, spawn_refresh_time=5) - await u.start_kernel() - await u.assert_code_output("5 * 4", "20", 5, 5) + assert await u.ensure_server_simulate(timeout=60, spawn_refresh_time=5) + assert await u.start_kernel() + assert await u.assert_code_output("5 * 4", "20", 5, 5) async def test_user_server_started_with_custom_base_url(): diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index cc9d766..e1d376c 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -1,4 +1,5 @@ """tests for the proxy""" + import os import shutil import ssl diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 055c9b1..ce01d0e 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -2,6 +2,7 @@ Test the plugin in integration-tests/plugins/simplest that makes use of all tljh recognized plugin hooks that are defined in tljh/hooks.py. """ + import os import subprocess @@ -19,7 +20,7 @@ def test_tljh_extra_user_conda_packages(): def test_tljh_extra_user_pip_packages(): - subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import django"]) + subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import simplejson"]) def test_tljh_extra_hub_pip_packages(): diff --git a/tests/conftest.py b/tests/conftest.py index cf765b9..106d534 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """pytest fixtures""" + import os import types from importlib import reload diff --git a/tests/test_conda.py b/tests/test_conda.py index de69f71..efe8ad0 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -1,6 +1,7 @@ """ Test conda commandline wrappers """ + import os import subprocess import tempfile @@ -13,9 +14,9 @@ from tljh import conda, installer @pytest.fixture(scope="module") def prefix(): """ - Provide a temporary directory with a mambaforge conda environment + Provide a temporary directory with a conda environment """ - installer_url, checksum = installer._mambaforge_url() + installer_url, checksum = installer._miniforge_url() with tempfile.TemporaryDirectory() as tmpdir: with conda.download_miniconda_installer( installer_url, checksum diff --git a/tests/test_installer.py b/tests/test_installer.py index 07081e4..f8db748 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -1,6 +1,7 @@ """ Unit test functions in installer.py """ + import json import os from subprocess import PIPE, run @@ -45,12 +46,12 @@ def test_ensure_admins(tljh_dir, admins, expected_config): def setup_conda(distro, version, prefix): - """Install mambaforge or miniconda in a prefix""" + """Install miniforge or miniconda in a prefix""" if distro == "mambaforge": - installer_url, _ = installer._mambaforge_url(version) + installer_url, _ = installer._miniforge_url(version) + installer_url = installer_url.replace("Miniforge3", "Mambaforge") elif distro == "miniforge": - installer_url, _ = installer._mambaforge_url(version) - installer_url = installer_url.replace("Mambaforge", "Miniforge3") + installer_url, _ = installer._miniforge_url(version) elif distro == "miniconda": arch = os.uname().machine installer_url = ( @@ -123,9 +124,9 @@ def _specifier(version): None, None, { - "python": "3.10.*", - "conda": "23.1.0", - "mamba": "1.4.1", + "python": "3.12.*", + "conda": "24.7.1", + "mamba": "1.5.9", }, ), # previous install, 1.0 diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 87275f6..91821a8 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -1,6 +1,7 @@ """ Unit test functions in installer.py """ + import os from datetime import date diff --git a/tests/test_normalize.py b/tests/test_normalize.py index fe7c623..d408c3f 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1,6 +1,7 @@ """ Test functions for normalizing various kinds of values """ + from tljh.normalize import generate_system_username diff --git a/tests/test_traefik.py b/tests/test_traefik.py index f950266..9b0c8f8 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -1,4 +1,5 @@ """Test traefik configuration""" + import os import pytest diff --git a/tests/test_user.py b/tests/test_user.py index 5bda68d..148e174 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,6 +1,7 @@ """ Test wrappers in tljw.user module """ + import grp import os import os.path diff --git a/tljh/apt.py b/tljh/apt.py index b5b0845..d746f45 100644 --- a/tljh/apt.py +++ b/tljh/apt.py @@ -1,6 +1,7 @@ """ Utilities for working with the apt package manager """ + import os import subprocess diff --git a/tljh/conda.py b/tljh/conda.py index 6b958ef..574ff99 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -1,6 +1,7 @@ """ Wrap conda commandline program """ + import contextlib import hashlib import json diff --git a/tljh/config.py b/tljh/config.py index d308e9e..d72b03e 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -154,92 +154,137 @@ def remove_item_from_config(config, property_path, value): return config_copy +def validate_config(config, validate): + """ + Validate changes to the config with tljh-config against the schema + """ + import jsonschema + + from .config_schema import config_schema + + try: + jsonschema.validate(instance=config, schema=config_schema) + except jsonschema.exceptions.ValidationError as e: + if validate: + print( + f"Config validation error: {e.message}.\n" + "You can still apply this change without validation by re-running your command with the --no-validate flag.\n" + "If you think this validation error is incorrect, please report it to https://github.com/jupyterhub/the-littlest-jupyterhub/issues." + ) + exit() + + def show_config(config_path): """ Pretty print config from given config_path """ - try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} - + config = get_current_config(config_path) yaml.dump(config, sys.stdout) -def set_config_value(config_path, key_path, value): +def set_config_value(config_path, key_path, value, validate=True): """ Set key at key_path in config_path to value """ - # FIXME: Have a file lock here - # FIXME: Validate schema here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = set_item_in_config(config, key_path, value) + validate_config(config, validate) - config = set_item_in_config(config, key_path, value) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) -def unset_config_value(config_path, key_path): +def unset_config_value(config_path, key_path, validate=True): """ Unset key at key_path in config_path """ - # FIXME: Have a file lock here - # FIXME: Validate schema here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = unset_item_from_config(config, key_path) + validate_config(config, validate) - config = unset_item_from_config(config, key_path) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) -def add_config_value(config_path, key_path, value): +def add_config_value(config_path, key_path, value, validate=True): """ Add value to list at key_path """ - # FIXME: Have a file lock here - # FIXME: Validate schema here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = add_item_to_config(config, key_path, value) + validate_config(config, validate) - config = add_item_to_config(config, key_path, value) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) -def remove_config_value(config_path, key_path, value): +def remove_config_value(config_path, key_path, value, validate=True): """ Remove value from list at key_path """ - # FIXME: Have a file lock here - # FIXME: Validate schema here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) + try: + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = remove_item_from_config(config, key_path, value) + validate_config(config, validate) + + with open(config_path, "w") as f: + yaml.dump(config, f) + + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) + + +def get_current_config(config_path): + """ + Retrieve the current config at config_path + """ try: with open(config_path) as f: - config = yaml.load(f) + return yaml.load(f) except FileNotFoundError: - config = {} - - config = remove_item_from_config(config, key_path, value) - - with open(config_path, "w") as f: - yaml.dump(config, f) + return {} def check_hub_ready(): + """ + Checks that hub is running. + """ from .configurer import load_config base_url = load_config()["base_url"] @@ -336,6 +381,18 @@ def main(argv=None): argparser.add_argument( "--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file" ) + + argparser.add_argument( + "--validate", action="store_true", help="Validate the TLJH config" + ) + argparser.add_argument( + "--no-validate", + dest="validate", + action="store_false", + help="Do not validate the TLJH config", + ) + argparser.set_defaults(validate=True) + subparsers = argparser.add_subparsers(dest="action") show_parser = subparsers.add_parser("show", help="Show current configuration") @@ -383,13 +440,19 @@ def main(argv=None): if args.action == "show": show_config(args.config_path) elif args.action == "set": - set_config_value(args.config_path, args.key_path, parse_value(args.value)) + set_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "unset": - unset_config_value(args.config_path, args.key_path) + unset_config_value(args.config_path, args.key_path, args.validate) elif args.action == "add-item": - add_config_value(args.config_path, args.key_path, parse_value(args.value)) + add_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "remove-item": - remove_config_value(args.config_path, args.key_path, parse_value(args.value)) + remove_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "reload": reload_component(args.component) else: diff --git a/tljh/config_schema.py b/tljh/config_schema.py new file mode 100644 index 0000000..0b12c8f --- /dev/null +++ b/tljh/config_schema.py @@ -0,0 +1,117 @@ +""" +The schema against which the TLJH config file can be validated. + +Validation occurs when changing values with tljh-config. +""" + +config_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Littlest JupyterHub YAML config file", + "definitions": { + "BaseURL": { + "type": "string", + }, + "Users": { + "type": "object", + "additionalProperties": False, + "properties": { + "extra_user_groups": {"type": "object", "items": {"type": "string"}}, + "allowed": {"type": "array", "items": {"type": "string"}}, + "banned": {"type": "array", "items": {"type": "string"}}, + "admin": {"type": "array", "items": {"type": "string"}}, + }, + }, + "Services": { + "type": "object", + "properties": { + "cull": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "timeout": {"type": "integer"}, + "every": {"type": "integer"}, + "concurrency": {"type": "integer"}, + "users": {"type": "boolean"}, + "max_age": {"type": "integer"}, + "remove_named_servers": {"type": "boolean"}, + }, + } + }, + }, + "HTTP": { + "type": "object", + "additionalProperties": False, + "properties": { + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + }, + }, + "HTTPS": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "tls": {"$ref": "#/definitions/TLS"}, + "letsencrypt": {"$ref": "#/definitions/LetsEncrypt"}, + }, + }, + "LetsEncrypt": { + "type": "object", + "additionalProperties": False, + "properties": { + "email": {"type": "string", "format": "email"}, + "domains": { + "type": "array", + "items": {"type": "string", "format": "hostname"}, + }, + "staging": {"type": "boolean"}, + }, + }, + "TLS": { + "type": "object", + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "cert": {"type": "string"}}, + }, + "Limits": { + "description": "User CPU and memory limits.", + "type": "object", + "additionalProperties": False, + "properties": {"memory": {"type": "string"}, "cpu": {"type": "integer"}}, + }, + "UserEnvironment": { + "type": "object", + "additionalProperties": False, + "properties": { + "default_app": { + "type": "string", + "enum": ["jupyterlab", "classic"], + "default": "jupyterlab", + } + }, + }, + "TraefikAPI": { + "type": "object", + "additionalProperties": False, + "properties": { + "ip": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "username": {"type": "string"}, + "password": {"type": "string"}, + }, + }, + }, + "properties": { + "additionalProperties": False, + "base_url": {"$ref": "#/definitions/BaseURL"}, + "user_environment": {"$ref": "#/definitions/UserEnvironment"}, + "users": {"$ref": "#/definitions/Users"}, + "limits": {"$ref": "#/definitions/Limits"}, + "https": {"$ref": "#/definitions/HTTPS"}, + "http": {"$ref": "#/definitions/HTTP"}, + "traefik_api": {"$ref": "#/definitions/TraefikAPI"}, + "services": {"$ref": "#/definitions/Services"}, + }, +} diff --git a/tljh/configurer.py b/tljh/configurer.py index 962cdde..7a58bbe 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -199,6 +199,14 @@ def update_userlists(c, config): """ users = config["users"] + if ( + not users["allowed"] + and config["auth"]["type"] == default["auth"]["type"] + and "allow_all" not in c.FirstUseAuthenticator + ): + # _default_ authenticator, enable allow_all if no users specified + c.FirstUseAuthenticator.allow_all = True + c.Authenticator.allowed_users = set(users["allowed"]) c.Authenticator.blocked_users = set(users["banned"]) c.Authenticator.admin_users = set(users["admin"]) diff --git a/tljh/hooks.py b/tljh/hooks.py index 151134d..d6200b5 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -1,6 +1,7 @@ """ Hook specifications that pluggy plugins can override """ + import pluggy hookspec = pluggy.HookspecMarker("tljh") diff --git a/tljh/installer.py b/tljh/installer.py index 95557fc..0930f7d 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -136,13 +136,13 @@ def ensure_usergroups(): f.write("Defaults exempt_group = jupyterhub-admins\n") -# Install mambaforge using an installer from +# Install miniforge using an installer from # https://github.com/conda-forge/miniforge/releases -MAMBAFORGE_VERSION = "23.1.0-1" +MINIFORGE_VERSION = "24.7.1-0" # sha256 checksums -MAMBAFORGE_CHECKSUMS = { - "aarch64": "d9d89c9e349369702171008d9ee7c5ce80ed420e5af60bd150a3db4bf674443a", - "x86_64": "cfb16c47dc2d115c8b114280aa605e322173f029fdb847a45348bf4bd23c62ab", +MINIFORGE_CHECKSUMS = { + "aarch64": "7a3372268b45679584043b4ba1e0318ee5027384a8d330f2d991b14d815d6a6d", + "x86_64": "b64f77042cf8eafd31ced64f9253a74fb85db63545fe167ba5756aea0e8125be", } # minimum versions of packages @@ -156,22 +156,22 @@ MINIMUM_VERSIONS = { } -def _mambaforge_url(version=MAMBAFORGE_VERSION, arch=None): - """Return (URL, checksum) for mambaforge download for a given version and arch +def _miniforge_url(version=MINIFORGE_VERSION, arch=None): + """Return (URL, checksum) for miniforge download for a given version and arch Default values provided for both version and arch """ if arch is None: arch = os.uname().machine - installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format( + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Miniforge3-{v}-Linux-{arch}.sh".format( v=version, arch=arch, ) # Check system architecture, set appropriate installer checksum - checksum = MAMBAFORGE_CHECKSUMS.get(arch) + checksum = MINIFORGE_CHECKSUMS.get(arch) if not checksum: raise ValueError( - f"Unsupported architecture: {arch}. TLJH only supports {','.join(MAMBAFORGE_CHECKSUMS.keys())}" + f"Unsupported architecture: {arch}. TLJH only supports {','.join(MINIFORGE_CHECKSUMS.keys())}" ) return installer_url, checksum @@ -198,7 +198,7 @@ def ensure_user_environment(user_requirements_txt_file): raise OSError(msg) logger.info("Downloading & setting up user environment...") - installer_url, installer_sha256 = _mambaforge_url() + installer_url, installer_sha256 = _miniforge_url() with conda.download_miniconda_installer( installer_url, installer_sha256 ) as installer_path: @@ -242,11 +242,10 @@ def ensure_user_environment(user_requirements_txt_file): ) to_upgrade.append(pkg) - # force reinstall conda/mamba to ensure a basically consistent env - # avoids issues with RemoveError: 'requests' is a dependency of conda - # only do this for 'old' conda versions known to have a problem - # we don't know how old, but we know 4.10 is affected and 23.1 is not - if not is_fresh_install and V(package_versions.get("conda", "0")) < V("23.1"): + # force reinstall conda/mamba to ensure conda doesn't raise error + # "RemoveError: 'requests' is a dependency of conda" later on when + # conda/mamba is used to install/upgrade something + if not is_fresh_install: # force-reinstall doesn't upgrade packages # it reinstalls them in-place # only reinstall packages already present diff --git a/tljh/log.py b/tljh/log.py index ed7eca4..7a0d5e7 100644 --- a/tljh/log.py +++ b/tljh/log.py @@ -1,4 +1,5 @@ """Setup tljh logging""" + import logging import os diff --git a/tljh/normalize.py b/tljh/normalize.py index 01d1777..4f4c9e1 100644 --- a/tljh/normalize.py +++ b/tljh/normalize.py @@ -1,6 +1,7 @@ """ Functions to normalize various inputs """ + import hashlib diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 2a9324d..3a9c004 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -8,13 +8,13 @@ # If a dependency is bumped to a new major version, we should make a major # version release of tljh. # -jupyterhub>=4.0.2,<5 +jupyterhub>=5.1.0,<6 jupyterhub-systemdspawner>=1.0.1,<2 jupyterhub-firstuseauthenticator>=1.0.0,<2 jupyterhub-nativeauthenticator>=1.2.0,<2 jupyterhub-ldapauthenticator>=1.3.2,<2 jupyterhub-tmpauthenticator>=1.0.0,<2 -oauthenticator>=16.0.4,<17 +oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 # pycurl is installed to improve reliability and performance for when JupyterHub @@ -26,3 +26,6 @@ jupyterhub-idle-culler>=1.2.1,<2 # ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # pycurl>=7.45.2,<8 + +# filelock is used to help us do atomic operations on config file(s) +filelock>=3.15.4,<4 diff --git a/tljh/systemd-units/jupyterhub.service b/tljh/systemd-units/jupyterhub.service index 0648830..0fb6a4c 100644 --- a/tljh/systemd-units/jupyterhub.service +++ b/tljh/systemd-units/jupyterhub.service @@ -18,7 +18,7 @@ Environment=TLJH_INSTALL_PREFIX={install_prefix} Environment=PATH={install_prefix}/hub/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin # Run upgrade-db before starting, in case Hub version has changed # This is a no-op when no db exists or no upgrades are needed -ExecStart={python_interpreter_path} -m jupyterhub.app -f {jupyterhub_config_path} --upgrade-db +ExecStart={python_interpreter_path} -m jupyterhub -f {jupyterhub_config_path} --upgrade-db [Install] # Start service when system boots diff --git a/tljh/systemd.py b/tljh/systemd.py index f274fab..6bfaf1b 100644 --- a/tljh/systemd.py +++ b/tljh/systemd.py @@ -3,6 +3,7 @@ Wraps systemctl to install, uninstall, start & stop systemd services. If we use a debian package instead, we can get rid of all this code. """ + import os import subprocess diff --git a/tljh/traefik.py b/tljh/traefik.py index 4ea0d49..a19a3de 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -1,4 +1,5 @@ """Traefik installation and setup""" + import hashlib import io import logging diff --git a/tljh/user.py b/tljh/user.py index f5ed4ca..b03903f 100644 --- a/tljh/user.py +++ b/tljh/user.py @@ -3,6 +3,7 @@ User management for tljh. Supports minimal user & group management """ + import grp import pwd import subprocess diff --git a/tljh/utils.py b/tljh/utils.py index 8ab1ca8..fceec70 100644 --- a/tljh/utils.py +++ b/tljh/utils.py @@ -1,6 +1,7 @@ """ Miscellaneous functions useful in at least two places unrelated to each other """ + import logging import re import subprocess diff --git a/tljh/yaml.py b/tljh/yaml.py index e51381e..daff06b 100644 --- a/tljh/yaml.py +++ b/tljh/yaml.py @@ -3,6 +3,7 @@ ensures the same yaml settings for reading/writing throughout tljh """ + from ruamel.yaml import YAML from ruamel.yaml.composer import Composer