From ed6ff645c2bb843ac5cf977d2c66b5e6ef10887c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 05:45:52 +0000 Subject: [PATCH 01/61] build(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/integration-test.yaml | 4 ++-- .github/workflows/unit-test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index feba78d..89055ef 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -53,7 +53,7 @@ jobs: extra_flags: --upgrade-from=0.2.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -116,7 +116,7 @@ jobs: distro_image: "ubuntu:22.04" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index ea7e1e2..4aa62b4 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -48,7 +48,7 @@ jobs: python_version: "3.10" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "${{ matrix.python_version }}" From caab46e08ad6e1cd17fdb4dd1e797224c9b070c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:35:43 +0000 Subject: [PATCH 02/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.10.1 → v3.14.0](https://github.com/asottile/pyupgrade/compare/v3.10.1...v3.14.0) - [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec7d213..db02266 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.14.0 hooks: - id: pyupgrade args: @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black From e8696338712898db4b6810518ecdada2b5af264e Mon Sep 17 00:00:00 2001 From: Pierre de Buyl Date: Fri, 27 Oct 2023 11:56:22 +0200 Subject: [PATCH 03/61] Fix URL syntax in nativeauth.md --- docs/howto/auth/nativeauth.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/auth/nativeauth.md b/docs/howto/auth/nativeauth.md index 370e2e2..22a56f7 100644 --- a/docs/howto/auth/nativeauth.md +++ b/docs/howto/auth/nativeauth.md @@ -42,4 +42,4 @@ tljh-config reload ## Optional features -More optional features are available on the `authenticator documentation ` +More optional features are available on the [authenticator documentation](https://native-authenticator.readthedocs.io/en/latest/) From 79134e3a2f1645bb215a5d14ce5321dbc0bf0097 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 20:28:01 +0000 Subject: [PATCH 04/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.14.0 → v3.15.0](https://github.com/asottile/pyupgrade/compare/v3.14.0...v3.15.0) - [github.com/psf/black: 23.9.1 → 23.10.1](https://github.com/psf/black/compare/23.9.1...23.10.1) - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db02266..4bf1dfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.14.0 + rev: v3.15.0 hooks: - id: pyupgrade args: @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: - id: black @@ -49,7 +49,7 @@ repos: # Misc... - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available hooks: # Autoformat: Makes sure files end in a newline and only a newline. From f1c7e2d90f6b19d788d60c60ffc3e1533abefd48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 05:51:55 +0000 Subject: [PATCH 05/61] build(deps): bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/integration-test.yaml | 4 ++-- .github/workflows/unit-test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 89055ef..58585a3 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -54,7 +54,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" @@ -117,7 +117,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 4aa62b4..a7aedac 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python_version }}" From 0f5a0ccbe03417fce30b132f36125d89eb410333 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 20:22:48 +0000 Subject: [PATCH 06/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.12.0 → 5.13.2](https://github.com/pycqa/isort/compare/5.12.0...5.13.2) - [github.com/psf/black: 23.10.1 → 23.12.1](https://github.com/psf/black/compare/23.10.1...23.12.1) - [github.com/pre-commit/mirrors-prettier: v3.0.3 → v4.0.0-alpha.8](https://github.com/pre-commit/mirrors-prettier/compare/v3.0.3...v4.0.0-alpha.8) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bf1dfe..a68f41a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,19 +31,19 @@ repos: # Autoformat: Python code - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.12.1 hooks: - id: black # Autoformat: markdown, yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier From 4abf3f003f4d7ce2d693bac5657cc1c04bc0d971 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 4 Jan 2024 02:28:28 +0100 Subject: [PATCH 07/61] Add optional oauthenticator dependency for AzureAD --- tljh/requirements-hub-env.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 2a9324d..62f39f4 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -14,7 +14,7 @@ jupyterhub-firstuseauthenticator>=1.0.0,<2 jupyterhub-nativeauthenticator>=1.2.0,<2 jupyterhub-ldapauthenticator>=1.3.2,<2 jupyterhub-tmpauthenticator>=1.0.0,<2 -oauthenticator>=16.0.4,<17 +oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 # pycurl is installed to improve reliability and performance for when JupyterHub From 723c4e756d58de31b2eef704a71aeff4f59f2cc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 05:26:45 +0000 Subject: [PATCH 08/61] build(deps): bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/unit-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index a7aedac..c2fcf70 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -93,4 +93,4 @@ jobs: run: pytest tests timeout-minutes: 15 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 From 1c9301e4d75bb78f016c5bddc686fda43811afd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 05:26:48 +0000 Subject: [PATCH 09/61] build(deps): bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/unit-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index a7aedac..427ce9f 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -70,7 +70,7 @@ jobs: # completion. Make sure to update the key to bust the cache # properly if you make a change that should influence it. - name: Load cached Python dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /srv/venv/ key: >- From d2924578034c4c8a20b013dfba3b25f0966e3c13 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 21:51:23 -0500 Subject: [PATCH 10/61] Initial config setup --- tljh/config-schema.json | 1 + tljh/config.py | 8 ++++++++ tljh/requirements-hub-env.txt | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 tljh/config-schema.json diff --git a/tljh/config-schema.json b/tljh/config-schema.json new file mode 100644 index 0000000..15591da --- /dev/null +++ b/tljh/config-schema.json @@ -0,0 +1 @@ +file:///home/bradfojb/Desktop/Personal-Repos/the-littlest-jupyterhub/schema.json \ No newline at end of file diff --git a/tljh/config.py b/tljh/config.py index d308e9e..09ecef0 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -154,6 +154,13 @@ def remove_item_from_config(config, property_path, value): return config_copy +def validate_config(config): + import json + import jsonschema + config_schema = json.load("config-schema.json") + jsonschema.validate(instance=config, schema=config_schema) + + def show_config(config_path): """ Pretty print config from given config_path @@ -180,6 +187,7 @@ def set_config_value(config_path, key_path, value): config = {} config = set_item_in_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 62f39f4..cde66e4 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -17,6 +17,8 @@ jupyterhub-tmpauthenticator>=1.0.0,<2 oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 +jsonschema + # pycurl is installed to improve reliability and performance for when JupyterHub # makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when # making requests over tornado's SimpleHTTPClient automatically if pycurl is From 78d4b7fbc4a4fe170a6cd2275865c88b02b0da02 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 22:52:16 -0500 Subject: [PATCH 11/61] Test config setup --- tljh/config-schema.json | 143 +++++++++++++++++++++++++++++++++- tljh/config.py | 25 +++--- tljh/requirements-hub-env.txt | 2 - 3 files changed, 158 insertions(+), 12 deletions(-) diff --git a/tljh/config-schema.json b/tljh/config-schema.json index 15591da..4563d19 100644 --- a/tljh/config-schema.json +++ b/tljh/config-schema.json @@ -1 +1,142 @@ -file:///home/bradfojb/Desktop/Personal-Repos/the-littlest-jupyterhub/schema.json \ No newline at end of file +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Littlest JupyterHub YAML config file", + "definitions": { + "Users": { + "type": "object", + "properties": { + "extra_user_groups": { + "type": "object", + "items": { + "type": "string" + } + }, + "allowed": { + "type": "array", + "items": { + "type": "string" + } + }, + "banned": { + "type": "array", + "items": { + "type": "string" + } + }, + "admin": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "HTTP": { + "type": "object", + "additionalProperties": false, + "properties": { + "address": { + "type": "string", + "format": "ipv4" + }, + "port": { + "type": "integer" + } + } + }, + "HTTPS": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "address": { + "type": "string", + "format": "ipv4" + }, + "port": { + "type": "integer" + }, + "tls": { + "$ref": "#/definitions/TLS" + }, + "letsencrypt": { + "$ref": "#/definitions/LetsEncrypt" + } + } + }, + "LetsEncrypt": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "type": "string", + "format": "email" + }, + "domains": { + "type": "array", + "items": { + "type": "string", + "format": "hostname" + } + } + } + }, + "TLS": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "type": "string" + }, + "cert": { + "type": "string" + } + }, + "required": ["key", "cert"] + }, + "Limits": { + "description": "User CPU and memory limits.", + "type": "object", + "additionalProperties": false, + "properties": { + "memory": { + "type": "string" + }, + "cpu": { + "type": "integer" + } + } + }, + "UserEnvironment": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_app": { + "type": "string", + "enum": ["jupyterlab", "classic"], + "default": "jupyterlab" + } + } + } + }, + "properties": { + "additionalProperties": false, + "user_environment": { + "$ref": "#/definitions/UserEnvironment" + }, + "users": { + "$ref": "#/definitions/Users" + }, + "limits": { + "$ref": "#/definitions/Limits" + }, + "https": { + "$ref": "#/definitions/HTTPS" + }, + "http": { + "$ref": "#/definitions/HTTP" + } + } +} diff --git a/tljh/config.py b/tljh/config.py index 09ecef0..8f0767a 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,10 +155,18 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json - import jsonschema - config_schema = json.load("config-schema.json") - jsonschema.validate(instance=config, schema=config_schema) + import json, jsonschema + + cwd = os.getcwd() + config_schema_file = os.path.join(cwd, "config-schema.json") + with open(config_schema_file) as f: + config_schema = json.load(f) + + try: + jsonschema.validate(instance=config, schema=config_schema) + except jsonschema.exceptions.ValidationError as e: + print(e) + exit() def show_config(config_path): @@ -179,14 +187,13 @@ def set_config_value(config_path, key_path, value): Set key at key_path in config_path to value """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) except FileNotFoundError: config = {} - config = set_item_in_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: @@ -198,7 +205,6 @@ def unset_config_value(config_path, key_path): Unset key at key_path in config_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -206,6 +212,7 @@ def unset_config_value(config_path, key_path): config = {} config = unset_item_from_config(config, key_path) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) @@ -216,7 +223,6 @@ def add_config_value(config_path, key_path, value): Add value to list at key_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -224,6 +230,7 @@ def add_config_value(config_path, key_path, value): config = {} config = add_item_to_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) @@ -234,7 +241,6 @@ def remove_config_value(config_path, key_path, value): Remove value from list at key_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -242,6 +248,7 @@ def remove_config_value(config_path, key_path, value): config = {} config = remove_item_from_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index cde66e4..62f39f4 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -17,8 +17,6 @@ jupyterhub-tmpauthenticator>=1.0.0,<2 oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 -jsonschema - # pycurl is installed to improve reliability and performance for when JupyterHub # makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when # making requests over tornado's SimpleHTTPClient automatically if pycurl is From fb01dea5e4feab9c122c014873d0473bf6083e7c Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:01:36 -0500 Subject: [PATCH 12/61] Working schema path --- tljh/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 8f0767a..9bcd200 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -157,8 +157,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): import json, jsonschema - cwd = os.getcwd() - config_schema_file = os.path.join(cwd, "config-schema.json") + pd = os.path.abspath(os.path.join(__file__, os.pardir)) + config_schema_file = os.path.join(pd, "config-schema.json") with open(config_schema_file) as f: config_schema = json.load(f) From ef5c6c56b79bde9bd5652dd49e4d708fdea77ba7 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:06:31 -0500 Subject: [PATCH 13/61] `print` human readable error message --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 9bcd200..dce1bae 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -165,7 +165,7 @@ def validate_config(config): try: jsonschema.validate(instance=config, schema=config_schema) except jsonschema.exceptions.ValidationError as e: - print(e) + print(e.message) exit() From 929536de7bdaf40125e4e035c28165ff5ff4c7a4 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:07:38 -0500 Subject: [PATCH 14/61] Run `pre-commit` hooks --- tljh/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index dce1bae..3dbbc87 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,7 +155,9 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json, jsonschema + import json + + import jsonschema pd = os.path.abspath(os.path.join(__file__, os.pardir)) config_schema_file = os.path.join(pd, "config-schema.json") From 5a0de137d2b8a63dbcc9b6c26b7770f7985dfe48 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:22:57 -0500 Subject: [PATCH 15/61] Do not allow `additionalProperties` for `Users` --- tljh/config-schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tljh/config-schema.json b/tljh/config-schema.json index 4563d19..d983dc3 100644 --- a/tljh/config-schema.json +++ b/tljh/config-schema.json @@ -4,6 +4,7 @@ "definitions": { "Users": { "type": "object", + "additionalProperties": false, "properties": { "extra_user_groups": { "type": "object", From 166eba67351bcfe1a6bc34b7c9a657ab644b6250 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 12:48:31 -0500 Subject: [PATCH 16/61] Switch from JSON to py --- tljh/config-schema.json | 143 ---------------------------------------- tljh/config.py | 10 +-- tljh/config_schema.py | 77 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 151 deletions(-) delete mode 100644 tljh/config-schema.json create mode 100644 tljh/config_schema.py diff --git a/tljh/config-schema.json b/tljh/config-schema.json deleted file mode 100644 index d983dc3..0000000 --- a/tljh/config-schema.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Littlest JupyterHub YAML config file", - "definitions": { - "Users": { - "type": "object", - "additionalProperties": false, - "properties": { - "extra_user_groups": { - "type": "object", - "items": { - "type": "string" - } - }, - "allowed": { - "type": "array", - "items": { - "type": "string" - } - }, - "banned": { - "type": "array", - "items": { - "type": "string" - } - }, - "admin": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "HTTP": { - "type": "object", - "additionalProperties": false, - "properties": { - "address": { - "type": "string", - "format": "ipv4" - }, - "port": { - "type": "integer" - } - } - }, - "HTTPS": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "address": { - "type": "string", - "format": "ipv4" - }, - "port": { - "type": "integer" - }, - "tls": { - "$ref": "#/definitions/TLS" - }, - "letsencrypt": { - "$ref": "#/definitions/LetsEncrypt" - } - } - }, - "LetsEncrypt": { - "type": "object", - "additionalProperties": false, - "properties": { - "email": { - "type": "string", - "format": "email" - }, - "domains": { - "type": "array", - "items": { - "type": "string", - "format": "hostname" - } - } - } - }, - "TLS": { - "type": "object", - "additionalProperties": false, - "properties": { - "key": { - "type": "string" - }, - "cert": { - "type": "string" - } - }, - "required": ["key", "cert"] - }, - "Limits": { - "description": "User CPU and memory limits.", - "type": "object", - "additionalProperties": false, - "properties": { - "memory": { - "type": "string" - }, - "cpu": { - "type": "integer" - } - } - }, - "UserEnvironment": { - "type": "object", - "additionalProperties": false, - "properties": { - "default_app": { - "type": "string", - "enum": ["jupyterlab", "classic"], - "default": "jupyterlab" - } - } - } - }, - "properties": { - "additionalProperties": false, - "user_environment": { - "$ref": "#/definitions/UserEnvironment" - }, - "users": { - "$ref": "#/definitions/Users" - }, - "limits": { - "$ref": "#/definitions/Limits" - }, - "https": { - "$ref": "#/definitions/HTTPS" - }, - "http": { - "$ref": "#/definitions/HTTP" - } - } -} diff --git a/tljh/config.py b/tljh/config.py index 3dbbc87..09ee92e 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,14 +155,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json - import jsonschema - - pd = os.path.abspath(os.path.join(__file__, os.pardir)) - config_schema_file = os.path.join(pd, "config-schema.json") - with open(config_schema_file) as f: - config_schema = json.load(f) + from config_schema import config_schema try: jsonschema.validate(instance=config, schema=config_schema) @@ -314,7 +308,7 @@ def parse_value(value_str): return float(value_str) elif value_str.lower() == "true": return True - elif value_str.lower() == "false": + elif value_str.lower() == "False": return False else: # it's a string diff --git a/tljh/config_schema.py b/tljh/config_schema.py new file mode 100644 index 0000000..b0595b3 --- /dev/null +++ b/tljh/config_schema.py @@ -0,0 +1,77 @@ +config_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Littlest JupyterHub YAML config file", + "definitions": { + "Users": { + "type": "object", + "additionalProperties": False, + "properties": { + "extra_user_groups": {"type": "object", "items": {"type": "string"}}, + "allowed": {"type": "array", "items": {"type": "string"}}, + "banned": {"type": "array", "items": {"type": "string"}}, + "admin": {"type": "array", "items": {"type": "string"}}, + }, + }, + "HTTP": { + "type": "object", + "additionalProperties": False, + "properties": { + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + }, + }, + "HTTPS": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "tls": {"$ref": "#/definitions/TLS"}, + "letsencrypt": {"$ref": "#/definitions/LetsEncrypt"}, + }, + }, + "LetsEncrypt": { + "type": "object", + "additionalProperties": False, + "properties": { + "email": {"type": "string", "format": "email"}, + "domains": { + "type": "array", + "items": {"type": "string", "format": "hostname"}, + }, + }, + }, + "TLS": { + "type": "object", + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "cert": {"type": "string"}}, + "required": ["key", "cert"], + }, + "Limits": { + "description": "User CPU and memory limits.", + "type": "object", + "additionalProperties": False, + "properties": {"memory": {"type": "string"}, "cpu": {"type": "integer"}}, + }, + "UserEnvironment": { + "type": "object", + "additionalProperties": False, + "properties": { + "default_app": { + "type": "string", + "enum": ["jupyterlab", "classic"], + "default": "jupyterlab", + } + }, + }, + }, + "properties": { + "additionalProperties": False, + "user_environment": {"$ref": "#/definitions/UserEnvironment"}, + "users": {"$ref": "#/definitions/Users"}, + "limits": {"$ref": "#/definitions/Limits"}, + "https": {"$ref": "#/definitions/HTTPS"}, + "http": {"$ref": "#/definitions/HTTP"}, + }, +} From fa363658dfef926f2b94af0be52787821812899d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:08:27 -0500 Subject: [PATCH 17/61] Update schema based on values in `configurer.py` --- tljh/config.py | 3 ++- tljh/config_schema.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 09ee92e..1c4967b 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -156,7 +156,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): import jsonschema - from config_schema import config_schema + + from .config_schema import config_schema try: jsonschema.validate(instance=config, schema=config_schema) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index b0595b3..2a0c5f0 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -2,6 +2,9 @@ config_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Littlest JupyterHub YAML config file", "definitions": { + "BaseURL": { + "type": "string", + }, "Users": { "type": "object", "additionalProperties": False, @@ -12,6 +15,26 @@ config_schema = { "admin": {"type": "array", "items": {"type": "string"}}, }, }, + "Services": { + "type": "object", + "properties": { + "cull": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "timeout": {"type": "integer"}, + "every": {"type": "integer"}, + "concurrency": {"type": "integer"}, + "users": { + "type": "boolean", + }, + "max_age": {"type": "integer"}, + "remove_named_servers": {"type": "boolean"}, + }, + } + }, + }, "HTTP": { "type": "object", "additionalProperties": False, @@ -65,13 +88,30 @@ config_schema = { } }, }, + "TraefikAPI": { + "type": "object", + "additionalProperties": False, + "properties": { + "ip": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "username": { + "type": "string", + }, + "password": { + "type": "string", + }, + }, + }, }, "properties": { "additionalProperties": False, + "base_url": {"$ref": "#/definitions/BaseURL"}, "user_environment": {"$ref": "#/definitions/UserEnvironment"}, "users": {"$ref": "#/definitions/Users"}, "limits": {"$ref": "#/definitions/Limits"}, "https": {"$ref": "#/definitions/HTTPS"}, "http": {"$ref": "#/definitions/HTTP"}, + "traefik_api": {"$ref": "#/definitions/TraefikAPI"}, + "services": {"$ref": "#/definitions/Services"}, }, } From d0c9aa263aea28ee9cf5e24fb9b0255c01df8ac1 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:18:39 -0500 Subject: [PATCH 18/61] Remove `TLS` required properties --- tljh/config_schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index 2a0c5f0..6480c7f 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -69,7 +69,6 @@ config_schema = { "type": "object", "additionalProperties": False, "properties": {"key": {"type": "string"}, "cert": {"type": "string"}}, - "required": ["key", "cert"], }, "Limits": { "description": "User CPU and memory limits.", From 4912cffe65c0150c1fc91de0ad83c7006e0e880d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:27:58 -0500 Subject: [PATCH 19/61] Fix `parse_value` --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 1c4967b..7083dcc 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -309,7 +309,7 @@ def parse_value(value_str): return float(value_str) elif value_str.lower() == "true": return True - elif value_str.lower() == "False": + elif value_str.lower() == "false": return False else: # it's a string From 90602674582419c6ba1cae46e2836390222cf076 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:39:17 -0500 Subject: [PATCH 20/61] Add `LetsEncrypt.staging`; run `pre-commit` --- tljh/config_schema.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index 6480c7f..e7ae2ba 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -26,9 +26,7 @@ config_schema = { "timeout": {"type": "integer"}, "every": {"type": "integer"}, "concurrency": {"type": "integer"}, - "users": { - "type": "boolean", - }, + "users": {"type": "boolean"}, "max_age": {"type": "integer"}, "remove_named_servers": {"type": "boolean"}, }, @@ -63,6 +61,7 @@ config_schema = { "type": "array", "items": {"type": "string", "format": "hostname"}, }, + "staging": {"type": "boolean"}, }, }, "TLS": { @@ -93,12 +92,8 @@ config_schema = { "properties": { "ip": {"type": "string", "format": "ipv4"}, "port": {"type": "integer"}, - "username": { - "type": "string", - }, - "password": { - "type": "string", - }, + "username": {"type": "string"}, + "password": {"type": "string"}, }, }, }, From 1f7d6d1c55e9bc9cc7defc66249523d9541e8a6a Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:52:31 -0500 Subject: [PATCH 21/61] Add doc string for `validate_config` --- tljh/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tljh/config.py b/tljh/config.py index 7083dcc..8537cb3 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,6 +155,9 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): + """ + Validate changes to the config with tljh-config against the schema + """ import jsonschema from .config_schema import config_schema From 9755938d7da277b7950c45a8a48a54095f5c4926 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 20:27:06 +0000 Subject: [PATCH 22/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.12.1 → 24.1.1](https://github.com/psf/black/compare/23.12.1...24.1.1) - [github.com/pycqa/flake8: 6.1.0 → 7.0.0](https://github.com/pycqa/flake8/compare/6.1.0...7.0.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a68f41a..1bd2717 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.1.1 hooks: - id: black @@ -64,7 +64,7 @@ repos: # Lint: Python code - repo: https://github.com/pycqa/flake8 - rev: "6.1.0" + rev: "7.0.0" hooks: - id: flake8 From b35851f2b7c1f6b3b3d7eff7006a5c9734550658 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 20:29:32 +0000 Subject: [PATCH 23/61] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- bootstrap/bootstrap.py | 1 + integration-tests/plugins/simplest/tljh_simplest.py | 1 + integration-tests/test_bootstrap.py | 1 + integration-tests/test_proxy.py | 1 + integration-tests/test_simplest_plugin.py | 1 + tests/conftest.py | 1 + tests/test_conda.py | 1 + tests/test_installer.py | 1 + tests/test_migrator.py | 1 + tests/test_normalize.py | 1 + tests/test_traefik.py | 1 + tests/test_user.py | 1 + tljh/apt.py | 1 + tljh/conda.py | 1 + tljh/hooks.py | 1 + tljh/log.py | 1 + tljh/normalize.py | 1 + tljh/systemd.py | 1 + tljh/traefik.py | 1 + tljh/user.py | 1 + tljh/utils.py | 1 + tljh/yaml.py | 1 + 22 files changed, 22 insertions(+) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index ac52039..f69aceb 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -42,6 +42,7 @@ Command line flags, from "bootstrap.py --help": can also pass a branch name such as 'main' or a commit hash. """ + import logging import multiprocessing import os diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index 015b504..ff433f8 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -1,6 +1,7 @@ """ Simplest plugin that exercises all the hooks defined in tljh/hooks.py. """ + from tljh.hooks import hookimpl diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index 09ae140..eac9c92 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -9,6 +9,7 @@ FIXME: The last test stands out and could be part of the other tests, and the first two could be more like unit tests. Ideally, this file is significantly reduced. """ + import concurrent.futures import os import subprocess diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index cc9d766..e1d376c 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -1,4 +1,5 @@ """tests for the proxy""" + import os import shutil import ssl diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 96eb0a5..9c2337b 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -2,6 +2,7 @@ Test the plugin in integration-tests/plugins/simplest that makes use of all tljh recognized plugin hooks that are defined in tljh/hooks.py. """ + import os import subprocess diff --git a/tests/conftest.py b/tests/conftest.py index cf765b9..106d534 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """pytest fixtures""" + import os import types from importlib import reload diff --git a/tests/test_conda.py b/tests/test_conda.py index 6d44595..397499e 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -1,6 +1,7 @@ """ Test conda commandline wrappers """ + import os import subprocess import tempfile diff --git a/tests/test_installer.py b/tests/test_installer.py index 07081e4..f06bb6e 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -1,6 +1,7 @@ """ Unit test functions in installer.py """ + import json import os from subprocess import PIPE, run diff --git a/tests/test_migrator.py b/tests/test_migrator.py index 87275f6..91821a8 100644 --- a/tests/test_migrator.py +++ b/tests/test_migrator.py @@ -1,6 +1,7 @@ """ Unit test functions in installer.py """ + import os from datetime import date diff --git a/tests/test_normalize.py b/tests/test_normalize.py index fe7c623..d408c3f 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1,6 +1,7 @@ """ Test functions for normalizing various kinds of values """ + from tljh.normalize import generate_system_username diff --git a/tests/test_traefik.py b/tests/test_traefik.py index f950266..9b0c8f8 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -1,4 +1,5 @@ """Test traefik configuration""" + import os import pytest diff --git a/tests/test_user.py b/tests/test_user.py index 5bda68d..148e174 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,6 +1,7 @@ """ Test wrappers in tljw.user module """ + import grp import os import os.path diff --git a/tljh/apt.py b/tljh/apt.py index b5b0845..d746f45 100644 --- a/tljh/apt.py +++ b/tljh/apt.py @@ -1,6 +1,7 @@ """ Utilities for working with the apt package manager """ + import os import subprocess diff --git a/tljh/conda.py b/tljh/conda.py index a543645..49737c4 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -1,6 +1,7 @@ """ Wrap conda commandline program """ + import contextlib import hashlib import json diff --git a/tljh/hooks.py b/tljh/hooks.py index ddaa2a3..8bdaf79 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -1,6 +1,7 @@ """ Hook specifications that pluggy plugins can override """ + import pluggy hookspec = pluggy.HookspecMarker("tljh") diff --git a/tljh/log.py b/tljh/log.py index ed7eca4..7a0d5e7 100644 --- a/tljh/log.py +++ b/tljh/log.py @@ -1,4 +1,5 @@ """Setup tljh logging""" + import logging import os diff --git a/tljh/normalize.py b/tljh/normalize.py index 01d1777..4f4c9e1 100644 --- a/tljh/normalize.py +++ b/tljh/normalize.py @@ -1,6 +1,7 @@ """ Functions to normalize various inputs """ + import hashlib diff --git a/tljh/systemd.py b/tljh/systemd.py index f274fab..6bfaf1b 100644 --- a/tljh/systemd.py +++ b/tljh/systemd.py @@ -3,6 +3,7 @@ Wraps systemctl to install, uninstall, start & stop systemd services. If we use a debian package instead, we can get rid of all this code. """ + import os import subprocess diff --git a/tljh/traefik.py b/tljh/traefik.py index 4ea0d49..a19a3de 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -1,4 +1,5 @@ """Traefik installation and setup""" + import hashlib import io import logging diff --git a/tljh/user.py b/tljh/user.py index f5ed4ca..b03903f 100644 --- a/tljh/user.py +++ b/tljh/user.py @@ -3,6 +3,7 @@ User management for tljh. Supports minimal user & group management """ + import grp import pwd import subprocess diff --git a/tljh/utils.py b/tljh/utils.py index 8ab1ca8..fceec70 100644 --- a/tljh/utils.py +++ b/tljh/utils.py @@ -1,6 +1,7 @@ """ Miscellaneous functions useful in at least two places unrelated to each other """ + import logging import re import subprocess diff --git a/tljh/yaml.py b/tljh/yaml.py index e51381e..daff06b 100644 --- a/tljh/yaml.py +++ b/tljh/yaml.py @@ -3,6 +3,7 @@ ensures the same yaml settings for reading/writing throughout tljh """ + from ruamel.yaml import YAML from ruamel.yaml.composer import Composer From 7fa4e2bcec97a6114c69dd5cf5df22759caf67af Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 24 Feb 2024 21:47:36 +0100 Subject: [PATCH 24/61] ci: add test for debian 12 --- .github/workflows/integration-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 58585a3..0712bd8 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -36,6 +36,9 @@ jobs: - name: "Debian 11, Py 3.9" distro_image: "debian:11" extra_flags: "" + - name: "Debian 12, Py 3.11" + distro_image: "debian:12" + extra_flags: "" - name: "Ubuntu 20.04, Py 3.8" distro_image: "ubuntu:20.04" extra_flags: "" From 10ba571bded6b185962364ae3eca6907c1b5bcb2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sun, 25 Feb 2024 15:42:07 +0100 Subject: [PATCH 25/61] ci: add test for ubuntu 24.04 --- .github/workflows/integration-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 0712bd8..6e63a6f 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -45,6 +45,9 @@ jobs: - name: "Ubuntu 22.04 Py 3.10" distro_image: "ubuntu:22.04" extra_flags: "" + - name: "Ubuntu 24.04 Py 3.12" + distro_image: "ubuntu:24.04" + extra_flags: "" - name: "Ubuntu 22.04, Py 3.10, from main" distro_image: "ubuntu:22.04" extra_flags: --upgrade-from=main From 2fe9912333c6d125fd0291bc7b0ffab30e8822ae Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sun, 25 Feb 2024 15:04:14 +0100 Subject: [PATCH 26/61] tests: test pip install plugin with no-dependency package This is a simplification to help ensure we don't run into issues like below because requests gets installed via pip. ``` RemoveError: 'requests' is a dependency of conda and cannot be removed from conda's operating environment. ``` --- integration-tests/plugins/simplest/tljh_simplest.py | 2 +- integration-tests/test_simplest_plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index ff433f8..eda8c1d 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -12,7 +12,7 @@ def tljh_extra_user_conda_packages(): @hookimpl def tljh_extra_user_pip_packages(): - return ["django"] + return ["simplejson"] @hookimpl diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 9c2337b..9830c47 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -20,7 +20,7 @@ def test_tljh_extra_user_conda_packages(): def test_tljh_extra_user_pip_packages(): - subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import django"]) + subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import simplejson"]) def test_tljh_extra_hub_pip_packages(): From 7d8a84860f29f4363ba6839dabf3916ea67a259a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sun, 25 Feb 2024 17:14:52 +0100 Subject: [PATCH 27/61] tests: show mamba list for the user environment before/after upgrade This can help us check if a package has been installed from PyPI via pip, or from conda-forge or other channel via conda/mamba. --- .github/integration-test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/integration-test.py b/.github/integration-test.py index fcb8768..73ef8ba 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -167,15 +167,23 @@ def run_test( command = f"python3 /srv/src/bootstrap/bootstrap.py --version={upgrade_from}" run_command(container_name, command) + # show user environment + command = "/opt/tljh/user/bin/mamba list" + run_command(container_name, command) + command = f"python3 /srv/src/bootstrap/bootstrap.py {' '.join(installer_args)}" run_command(container_name, command) + # show user environment (again if upgrade) + command = "/opt/tljh/user/bin/mamba list" + run_command(container_name, command) + # Install pkgs from requirements in hub's pip, where # the bootstrap script installed the others command = "/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt" run_command(container_name, command) - # show environment + # show hub environment command = "/opt/tljh/hub/bin/python3 -m pip freeze" run_command(container_name, command) From 4e397bc687b73c68ee96557db2daddcbb13f66c6 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 26 Feb 2024 09:40:00 +0100 Subject: [PATCH 28/61] Re-install conda/mamba for every tljh upgrade (doesn't imply upgrade) --- tljh/installer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index aff24fc..5a9d767 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -242,11 +242,10 @@ def ensure_user_environment(user_requirements_txt_file): ) to_upgrade.append(pkg) - # force reinstall conda/mamba to ensure a basically consistent env - # avoids issues with RemoveError: 'requests' is a dependency of conda - # only do this for 'old' conda versions known to have a problem - # we don't know how old, but we know 4.10 is affected and 23.1 is not - if not is_fresh_install and V(package_versions.get("conda", "0")) < V("23.1"): + # force reinstall conda/mamba to ensure conda doesn't raise error + # "RemoveError: 'requests' is a dependency of conda" later on when + # conda/mamba is used to install/upgrade something + if not is_fresh_install: # force-reinstall doesn't upgrade packages # it reinstalls them in-place # only reinstall packages already present From d9a0a5fc7ab806ab2a22926a843f7aff73d20c90 Mon Sep 17 00:00:00 2001 From: David Alber Date: Fri, 1 Mar 2024 11:50:47 -0800 Subject: [PATCH 29/61] Fix typo and replace word: "disitnguishing" --> "distinctive" The typo is of "distinguishing", but "distinctive" makes more sense to me in the context of the sentence it is in. --- docs/install/amazon.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install/amazon.md b/docs/install/amazon.md index dd042be..8677298 100644 --- a/docs/install/amazon.md +++ b/docs/install/amazon.md @@ -139,7 +139,7 @@ Let's create the server on which we can run JupyterHub. SSH to connect (port 22). If you have never used your Amazon account before, you'll have to select - **Create a new security group**. You should give it a disitnguishing name + **Create a new security group**. You should give it a distinctive name under **Security group name** such as `ssh_web` for future reference. If you have, one from before you can select it and adjust it to have the rules you need, if you prefer. From beed70060c7b55d191f7b58db993249a66bd4bbe Mon Sep 17 00:00:00 2001 From: David Alber Date: Fri, 1 Mar 2024 12:09:17 -0800 Subject: [PATCH 30/61] Reword sentence --- docs/install/amazon.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install/amazon.md b/docs/install/amazon.md index dd042be..6dd0859 100644 --- a/docs/install/amazon.md +++ b/docs/install/amazon.md @@ -141,8 +141,8 @@ Let's create the server on which we can run JupyterHub. If you have never used your Amazon account before, you'll have to select **Create a new security group**. You should give it a disitnguishing name under **Security group name** - such as `ssh_web` for future reference. If you have, one from before you can - select it and adjust it to have the rules you need, if you prefer. + such as `ssh_web` for future reference. If you already have a security group, + you can select it and adjust it to have the rules you need, if you prefer. The rules will default to include `SSH`. Leave that there, and then click on the **Add Rule** button. Under **Type** for the new rule, change the field From 48fe4403726417209fac2f862fc6d5048ead0dfe Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:36:49 -0400 Subject: [PATCH 31/61] Add `--[no-]validation` flag for `tljh-config` --- tljh/config.py | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 8537cb3..849fdb5 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -154,7 +154,7 @@ def remove_item_from_config(config, property_path, value): return config_copy -def validate_config(config): +def validate_config(config, validate): """ Validate changes to the config with tljh-config against the schema """ @@ -165,8 +165,13 @@ def validate_config(config): try: jsonschema.validate(instance=config, schema=config_schema) except jsonschema.exceptions.ValidationError as e: - print(e.message) - exit() + if validate: + print( + f"Config validation error: {e.message}.\n" + "You can still apply this change without validation by re-running your command with the --no-validate flag.\n" + "If you think this validation error is incorrect, please report it to https://github.com/jupyterhub/the-littlest-jupyterhub/issues." + ) + exit() def show_config(config_path): @@ -182,7 +187,7 @@ def show_config(config_path): yaml.dump(config, sys.stdout) -def set_config_value(config_path, key_path, value): +def set_config_value(config_path, key_path, value, validate): """ Set key at key_path in config_path to value """ @@ -194,13 +199,13 @@ def set_config_value(config_path, key_path, value): config = {} config = set_item_in_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def unset_config_value(config_path, key_path): +def unset_config_value(config_path, key_path, validate): """ Unset key at key_path in config_path """ @@ -212,13 +217,13 @@ def unset_config_value(config_path, key_path): config = {} config = unset_item_from_config(config, key_path) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def add_config_value(config_path, key_path, value): +def add_config_value(config_path, key_path, value, validate): """ Add value to list at key_path """ @@ -230,13 +235,13 @@ def add_config_value(config_path, key_path, value): config = {} config = add_item_to_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def remove_config_value(config_path, key_path, value): +def remove_config_value(config_path, key_path, value, validate): """ Remove value from list at key_path """ @@ -248,7 +253,7 @@ def remove_config_value(config_path, key_path, value): config = {} config = remove_item_from_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) @@ -351,6 +356,12 @@ def main(argv=None): argparser.add_argument( "--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file" ) + argparser.add_argument( + "--validate", + action=argparse.BooleanOptionalAction, + help="Validate the TLJH config", + ) + subparsers = argparser.add_subparsers(dest="action") show_parser = subparsers.add_parser("show", help="Show current configuration") @@ -395,16 +406,25 @@ def main(argv=None): args = argparser.parse_args(argv) + if args.validate == None: + args.validate = True + if args.action == "show": show_config(args.config_path) elif args.action == "set": - set_config_value(args.config_path, args.key_path, parse_value(args.value)) + set_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "unset": - unset_config_value(args.config_path, args.key_path) + unset_config_value(args.config_path, args.key_path, args.validate) elif args.action == "add-item": - add_config_value(args.config_path, args.key_path, parse_value(args.value)) + add_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "remove-item": - remove_config_value(args.config_path, args.key_path, parse_value(args.value)) + remove_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "reload": reload_component(args.component) else: From 743f729902ec14d22f81420f58f9772077982d36 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:45:36 -0400 Subject: [PATCH 32/61] Update `set_config_value` tests --- tests/test_traefik.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index f950266..380ac4c 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,12 +64,12 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" + config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org", True ) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"] + config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"], True ) traefik.ensure_traefik_config(str(state_dir)) @@ -138,9 +138,13 @@ def test_letsencrypt_config(tljh_dir): def test_manual_ssl_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) - config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") - config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value( + config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True + ) + config.set_config_value( + config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True + ) traefik.ensure_traefik_config(str(state_dir)) cfg = _read_static_config(state_dir) @@ -244,12 +248,16 @@ def test_extra_config(tmpdir, tljh_dir): def test_listen_address(tmpdir, tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) - config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") - config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value( + config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True + ) + config.set_config_value( + config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True + ) - config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1") - config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1") + config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1", True) + config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1", True) traefik.ensure_traefik_config(str(state_dir)) From 46e404556801c978a8218957f28cacbf8f796818 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:08:21 -0400 Subject: [PATCH 33/61] Update `test_proxy` tests --- integration-tests/test_proxy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index cc9d766..f758655 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -63,9 +63,9 @@ def test_manual_https(preserve_config): "/CN=tljh.jupyer.org", ] ) - set_config_value(CONFIG_FILE, "https.enabled", True) - set_config_value(CONFIG_FILE, "https.tls.key", key) - set_config_value(CONFIG_FILE, "https.tls.cert", cert) + set_config_value(CONFIG_FILE, "https.enabled", True, True) + set_config_value(CONFIG_FILE, "https.tls.key", key, True) + set_config_value(CONFIG_FILE, "https.tls.cert", cert, True) reload_component("proxy") for i in range(10): time.sleep(i) @@ -89,7 +89,7 @@ def test_manual_https(preserve_config): # cleanup shutil.rmtree(ssl_dir) - set_config_value(CONFIG_FILE, "https.enabled", False) + set_config_value(CONFIG_FILE, "https.enabled", False, True) reload_component("proxy") From f921acc1833ccca2064663b1369538c446feecf8 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:09:07 -0400 Subject: [PATCH 34/61] Bump `ubuntu` image to `22.04` --- integration-tests/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/Dockerfile b/integration-tests/Dockerfile index c1c73d8..eb116eb 100644 --- a/integration-tests/Dockerfile +++ b/integration-tests/Dockerfile @@ -1,5 +1,5 @@ # Systemd inside a Docker container, for CI only -ARG BASE_IMAGE=ubuntu:20.04 +ARG BASE_IMAGE=ubuntu:22.04 FROM $BASE_IMAGE # DEBIAN_FRONTEND is set to avoid being asked for input and hang during build: @@ -29,8 +29,8 @@ RUN systemctl set-default multi-user.target STOPSIGNAL SIGRTMIN+3 # Uncomment these lines for a development install -#ENV TLJH_BOOTSTRAP_DEV=yes -#ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src -#ENV PATH=/opt/tljh/hub/bin:${PATH} +# ENV TLJH_BOOTSTRAP_DEV=yes +# ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src +# ENV PATH=/opt/tljh/hub/bin:${PATH} CMD ["/bin/bash", "-c", "exec /lib/systemd/systemd --log-target=journal 3>&1"] From 4ddd798928a25d3fc5d8faad441eebfb2d45b1f2 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:08:28 -0400 Subject: [PATCH 35/61] Add docstring from `config_schema.py` --- tljh/config_schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index e7ae2ba..0b12c8f 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -1,3 +1,9 @@ +""" +The schema against which the TLJH config file can be validated. + +Validation occurs when changing values with tljh-config. +""" + config_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Littlest JupyterHub YAML config file", From 5169301a0a507aadc858487ab2e3bb84888c01ac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 22:10:42 +0000 Subject: [PATCH 36/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.15.0 → v3.15.2](https://github.com/asottile/pyupgrade/compare/v3.15.0...v3.15.2) - [github.com/PyCQA/autoflake: v2.2.1 → v2.3.1](https://github.com/PyCQA/autoflake/compare/v2.2.1...v2.3.1) - [github.com/psf/black: 24.1.1 → 24.3.0](https://github.com/psf/black/compare/24.1.1...24.3.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bd2717..af94704 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.15.2 hooks: - id: pyupgrade args: @@ -22,7 +22,7 @@ repos: # Autoformat: Python code - repo: https://github.com/PyCQA/autoflake - rev: v2.2.1 + rev: v2.3.1 hooks: - id: autoflake # args ref: https://github.com/PyCQA/autoflake#advanced-usage @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.3.0 hooks: - id: black From 196208ae58bf83b4c400d549023d8f936eece5b1 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 13:30:32 -0400 Subject: [PATCH 37/61] Remove `argparse.BooleanOptionalAction` for Python 3.8 compatibility --- tljh/config.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 849fdb5..3f942c8 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -356,11 +356,17 @@ def main(argv=None): argparser.add_argument( "--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file" ) + argparser.add_argument( - "--validate", - action=argparse.BooleanOptionalAction, - help="Validate the TLJH config", + "--validate", action="store_true", help="Validate the TLJH config" ) + argparser.add_argument( + "--no-validate", + dest="validate", + action="store_false", + help="Do not validate the TLJH config", + ) + argparser.set_defaults(validate=True) subparsers = argparser.add_subparsers(dest="action") From 51f847053510e239c92b3d24793dfe38a3dab321 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:05:32 -0400 Subject: [PATCH 38/61] Fix unit tests with `pip` --- .github/workflows/unit-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 3b5f857..ae77c29 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -59,6 +59,7 @@ jobs: apt-get update apt-get install --yes \ python3-venv \ + python3-pip \ bzip2 \ git From c578a7bec09612eb010e18866ea4631387e6dcee Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:10:16 -0400 Subject: [PATCH 39/61] Update `step` `name` --- .github/workflows/unit-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index ae77c29..154bd1c 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -53,7 +53,7 @@ jobs: with: python-version: "${{ matrix.python_version }}" - - name: Install venv, git and setup venv + - name: Install venv, git, pip and setup venv run: | export DEBIAN_FRONTEND=noninteractive apt-get update From 67dd3c8abe985253b7da402ff195ad4429232c54 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:02:55 -0400 Subject: [PATCH 40/61] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 3f942c8..6dc2ed2 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -223,7 +223,7 @@ def unset_config_value(config_path, key_path, validate): yaml.dump(config, f) -def add_config_value(config_path, key_path, value, validate): +def add_config_value(config_path, key_path, value, validate=True): """ Add value to list at key_path """ From b94a281ff87d751c69fdd2aada5a6f8e65ccb67a Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:03:05 -0400 Subject: [PATCH 41/61] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 6dc2ed2..5911703 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -187,7 +187,7 @@ def show_config(config_path): yaml.dump(config, sys.stdout) -def set_config_value(config_path, key_path, value, validate): +def set_config_value(config_path, key_path, value, validate=True): """ Set key at key_path in config_path to value """ From 5ae31ce169327cf9713394f93651b700b3e4abe4 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:03:14 -0400 Subject: [PATCH 42/61] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 5911703..1f68c83 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -205,7 +205,7 @@ def set_config_value(config_path, key_path, value, validate=True): yaml.dump(config, f) -def unset_config_value(config_path, key_path, validate): +def unset_config_value(config_path, key_path, validate=True): """ Unset key at key_path in config_path """ From 7474b876f1f8647b68559d71e94ba6d99e27ba1a Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:04:29 -0400 Subject: [PATCH 43/61] Remove unneeded `validate` code --- tljh/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 1f68c83..b496400 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -412,9 +412,6 @@ def main(argv=None): args = argparser.parse_args(argv) - if args.validate == None: - args.validate = True - if args.action == "show": show_config(args.config_path) elif args.action == "set": From 9bcfa7032697a74befebda43e444efde1a3fb087 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:05:16 -0400 Subject: [PATCH 44/61] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index b496400..633cc36 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -241,7 +241,7 @@ def add_config_value(config_path, key_path, value, validate=True): yaml.dump(config, f) -def remove_config_value(config_path, key_path, value, validate): +def remove_config_value(config_path, key_path, value, validate=True): """ Remove value from list at key_path """ From 38a01e840689e1ee526259f2b32ba2cb123ae690 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:09:18 -0400 Subject: [PATCH 45/61] Use default `true` for `validate` --- integration-tests/test_proxy.py | 8 ++++---- tests/test_traefik.py | 30 +++++++++++------------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index f758655..cc9d766 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -63,9 +63,9 @@ def test_manual_https(preserve_config): "/CN=tljh.jupyer.org", ] ) - set_config_value(CONFIG_FILE, "https.enabled", True, True) - set_config_value(CONFIG_FILE, "https.tls.key", key, True) - set_config_value(CONFIG_FILE, "https.tls.cert", cert, True) + set_config_value(CONFIG_FILE, "https.enabled", True) + set_config_value(CONFIG_FILE, "https.tls.key", key) + set_config_value(CONFIG_FILE, "https.tls.cert", cert) reload_component("proxy") for i in range(10): time.sleep(i) @@ -89,7 +89,7 @@ def test_manual_https(preserve_config): # cleanup shutil.rmtree(ssl_dir) - set_config_value(CONFIG_FILE, "https.enabled", False, True) + set_config_value(CONFIG_FILE, "https.enabled", False) reload_component("proxy") diff --git a/tests/test_traefik.py b/tests/test_traefik.py index 380ac4c..b060be4 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,12 +64,12 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value(config.CONFIG_FILE, "https.enabled") config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org", True + config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" ) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"], True + config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"] ) traefik.ensure_traefik_config(str(state_dir)) @@ -138,13 +138,9 @@ def test_letsencrypt_config(tljh_dir): def test_manual_ssl_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) - config.set_config_value( - config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True - ) - config.set_config_value( - config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True - ) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") + config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") traefik.ensure_traefik_config(str(state_dir)) cfg = _read_static_config(state_dir) @@ -248,16 +244,12 @@ def test_extra_config(tmpdir, tljh_dir): def test_listen_address(tmpdir, tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) - config.set_config_value( - config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True - ) - config.set_config_value( - config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True - ) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") + config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") - config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1", True) - config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1", True) + config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1") + config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1") traefik.ensure_traefik_config(str(state_dir)) From 5469e21e74bcddd8731bac947ef7331a030ede5d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:19:45 -0400 Subject: [PATCH 46/61] Fix removal of `https.enabled` --- tests/test_traefik.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index b060be4..f950266 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,7 +64,7 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) config.set_config_value( config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" ) From 8fd41cc77af4c9c8aa6f08b42c54751d93307fbe Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 3 Apr 2024 22:17:23 +0200 Subject: [PATCH 47/61] tests: strengthen tests to fail earlier on issues --- integration-tests/test_hub.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 55a35cd..9281566 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -59,9 +59,9 @@ async def test_user_code_execute(): async with User(username, HUB_URL, partial(login_dummy, password="")) as u: assert await u.login() - await u.ensure_server_simulate(timeout=60, spawn_refresh_time=5) - await u.start_kernel() - await u.assert_code_output("5 * 4", "20", 5, 5) + assert await u.ensure_server_simulate(timeout=60, spawn_refresh_time=5) + assert await u.start_kernel() + assert await u.assert_code_output("5 * 4", "20", 5, 5) async def test_user_server_started_with_custom_base_url(): From bdbb3adbbc1670b5f8f0fa7a4e762c249d6a592c Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:49:07 -0400 Subject: [PATCH 48/61] Add config lockfile --- tljh/config.py | 109 +++++++++++++++++++++------------- tljh/requirements-hub-env.txt | 2 + 2 files changed, 69 insertions(+), 42 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 633cc36..d72b03e 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -178,12 +178,7 @@ def show_config(config_path): """ Pretty print config from given config_path """ - try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} - + config = get_current_config(config_path) yaml.dump(config, sys.stdout) @@ -191,75 +186,105 @@ def set_config_value(config_path, key_path, value, validate=True): """ Set key at key_path in config_path to value """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} - config = set_item_in_config(config, key_path, value) + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = set_item_in_config(config, key_path, value) + validate_config(config, validate) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def unset_config_value(config_path, key_path, validate=True): """ Unset key at key_path in config_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = unset_item_from_config(config, key_path) + validate_config(config, validate) - config = unset_item_from_config(config, key_path) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def add_config_value(config_path, key_path, value, validate=True): """ Add value to list at key_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = add_item_to_config(config, key_path, value) + validate_config(config, validate) - config = add_item_to_config(config, key_path, value) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def remove_config_value(config_path, key_path, value, validate=True): """ Remove value from list at key_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) + try: + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = remove_item_from_config(config, key_path, value) + validate_config(config, validate) + + with open(config_path, "w") as f: + yaml.dump(config, f) + + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) + + +def get_current_config(config_path): + """ + Retrieve the current config at config_path + """ try: with open(config_path) as f: - config = yaml.load(f) + return yaml.load(f) except FileNotFoundError: - config = {} - - config = remove_item_from_config(config, key_path, value) - validate_config(config, validate) - - with open(config_path, "w") as f: - yaml.dump(config, f) + return {} def check_hub_ready(): + """ + Checks that hub is running. + """ from .configurer import load_config base_url = load_config()["base_url"] diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 62f39f4..556ad19 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -26,3 +26,5 @@ jupyterhub-idle-culler>=1.2.1,<2 # ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # pycurl>=7.45.2,<8 + +filelock>=3.13.3,<4 From 13ed32b499c297f4cf9e7ecd0e3abcd89a0597ef Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:51:51 -0400 Subject: [PATCH 49/61] Add `lockfile` to integration tests --- integration-tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index c086c7f..3ef31e4 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,3 +1,4 @@ +filelock pytest pytest-cov pytest-asyncio From 96ac0d3538b1a21c04b712dd1035e0412619d210 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 11:03:24 -0400 Subject: [PATCH 50/61] Add `filelock` to `dev-requirements` --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 672ad32..f3b24c9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ +filelock packaging pytest pytest-cov From 52478abb657b050e2697c83be527f4320c03645a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 22:03:37 +0000 Subject: [PATCH 51/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 24.3.0 → 24.4.2](https://github.com/psf/black/compare/24.3.0...24.4.2) - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af94704..e024edb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.4.2 hooks: - id: black @@ -49,7 +49,7 @@ repos: # Misc... - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available hooks: # Autoformat: Makes sure files end in a newline and only a newline. From a5f966927fecf365515c6b939ec6b6a4bf74167e Mon Sep 17 00:00:00 2001 From: Joseph Daudi Date: Thu, 23 May 2024 21:49:15 +0300 Subject: [PATCH 52/61] Added missing details on how to add custom domain form manual HTTPS configuration --- docs/howto/admin/https.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/howto/admin/https.md b/docs/howto/admin/https.md index 1cdb459..449ef41 100644 --- a/docs/howto/admin/https.md +++ b/docs/howto/admin/https.md @@ -89,6 +89,7 @@ If so, you can tell your deployment to use these files: sudo tljh-config set https.enabled true sudo tljh-config set https.tls.key /etc/mycerts/mydomain.key sudo tljh-config set https.tls.cert /etc/mycerts/mydomain.cert +sudo tljh-config add-item https.tls.domains yourhub.yourdomain.edu ``` Once you have loaded this, your config should look like: @@ -103,6 +104,8 @@ https: tls: key: /etc/mycerts/mydomain.key cert: /etc/mycerts/mydomain.cert + domains: + - yourhub.yourdomain.edu ``` Finally, you can reload the proxy to load the new configuration: From 9c7427923d006a2ffb8acb23bf2300fd28af1d3f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 00:12:39 +0000 Subject: [PATCH 53/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.15.2 → v3.16.0](https://github.com/asottile/pyupgrade/compare/v3.15.2...v3.16.0) - [github.com/pycqa/flake8: 7.0.0 → 7.1.0](https://github.com/pycqa/flake8/compare/7.0.0...7.1.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e024edb..7df0a05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.16.0 hooks: - id: pyupgrade args: @@ -64,7 +64,7 @@ repos: # Lint: Python code - repo: https://github.com/pycqa/flake8 - rev: "7.0.0" + rev: "7.1.0" hooks: - id: flake8 From 708bbec2a4061d0f3df9bed1db637d163593ff46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:07:01 +0000 Subject: [PATCH 54/61] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.16.0 → v3.17.0](https://github.com/asottile/pyupgrade/compare/v3.16.0...v3.17.0) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) - [github.com/pycqa/flake8: 7.1.0 → 7.1.1](https://github.com/pycqa/flake8/compare/7.1.0...7.1.1) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7df0a05..688a10e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: # Autoformat: Python code, syntax patterns are modernized - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.17.0 hooks: - id: pyupgrade args: @@ -37,7 +37,7 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black @@ -64,7 +64,7 @@ repos: # Lint: Python code - repo: https://github.com/pycqa/flake8 - rev: "7.1.0" + rev: "7.1.1" hooks: - id: flake8 From 5abf657bdf3cce1ce7f43733932318a02c68b119 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Aug 2024 08:32:24 +0200 Subject: [PATCH 55/61] fix `-m` invocation of jupyterhub -m jupyterhub.app results in multiple JupyterHub classes being defined and jupyterhub.app.JupyterHub never being instantiated --- docs/howto/admin/systemd.md | 2 +- tljh/systemd-units/jupyterhub.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/admin/systemd.md b/docs/howto/admin/systemd.md index 468cea7..f1716d9 100644 --- a/docs/howto/admin/systemd.md +++ b/docs/howto/admin/systemd.md @@ -23,7 +23,7 @@ PrivateDevices=yes ProtectKernelTunables=yes ProtectKernelModules=yes Environment=TLJH_INSTALL_PREFIX=/opt/tljh -ExecStart=/opt/tljh/hub/bin/python3 -m jupyterhub.app -f jupyterhub_config.py --upgrade-db +ExecStart=/opt/tljh/hub/bin/python3 -m jupyterhub -f jupyterhub_config.py --upgrade-db [Install] WantedBy=multi-user.target diff --git a/tljh/systemd-units/jupyterhub.service b/tljh/systemd-units/jupyterhub.service index 0648830..0fb6a4c 100644 --- a/tljh/systemd-units/jupyterhub.service +++ b/tljh/systemd-units/jupyterhub.service @@ -18,7 +18,7 @@ Environment=TLJH_INSTALL_PREFIX={install_prefix} Environment=PATH={install_prefix}/hub/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin # Run upgrade-db before starting, in case Hub version has changed # This is a no-op when no db exists or no upgrades are needed -ExecStart={python_interpreter_path} -m jupyterhub.app -f {jupyterhub_config_path} --upgrade-db +ExecStart={python_interpreter_path} -m jupyterhub -f {jupyterhub_config_path} --upgrade-db [Install] # Start service when system boots From 65c5d78ea57327c53bb18d84a5e7a54b4a6ef226 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Aug 2024 09:52:11 +0200 Subject: [PATCH 56/61] jupyterhub 5 require 5.1.0 for security fixes --- integration-tests/test_hub.py | 2 +- tljh/requirements-hub-env.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 9281566..939175d 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -33,7 +33,7 @@ def test_hub_version(): r = requests.get(HUB_URL + "/hub/api") r.raise_for_status() info = r.json() - assert V("4") <= V(info["version"]) <= V("5") + assert V("5.1") <= V(info["version"]) <= V("6") async def test_user_code_execute(): diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 62f39f4..c3b1887 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -8,7 +8,7 @@ # If a dependency is bumped to a new major version, we should make a major # version release of tljh. # -jupyterhub>=4.0.2,<5 +jupyterhub>=5.1.0,<6 jupyterhub-systemdspawner>=1.0.1,<2 jupyterhub-firstuseauthenticator>=1.0.0,<2 jupyterhub-nativeauthenticator>=1.2.0,<2 From c492c176bc1a2be9ff5274da69a4250fe98bf4ca Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Aug 2024 11:36:09 +0200 Subject: [PATCH 57/61] allow all by default with default FirstUseAuthenticator doesn't take effect if there's any auth config --- tljh/configurer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tljh/configurer.py b/tljh/configurer.py index 962cdde..7a58bbe 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -199,6 +199,14 @@ def update_userlists(c, config): """ users = config["users"] + if ( + not users["allowed"] + and config["auth"]["type"] == default["auth"]["type"] + and "allow_all" not in c.FirstUseAuthenticator + ): + # _default_ authenticator, enable allow_all if no users specified + c.FirstUseAuthenticator.allow_all = True + c.Authenticator.allowed_users = set(users["allowed"]) c.Authenticator.blocked_users = set(users["banned"]) c.Authenticator.admin_users = set(users["admin"]) From dcbb37688e7be4b2324094dd6af1d57d468f8b4d Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Aug 2024 11:39:41 +0200 Subject: [PATCH 58/61] doc adding users with github, since they won't be allowed by default --- docs/howto/auth/github.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/howto/auth/github.md b/docs/howto/auth/github.md index 47fed0a..2e5baf5 100644 --- a/docs/howto/auth/github.md +++ b/docs/howto/auth/github.md @@ -91,10 +91,16 @@ For more information on `tljh-config`, see [](/topic/tljh-config). 4. Tell your JupyterHub to _use_ the GitHub OAuthenticator for authentication: ``` - sudo tljh-config set auth.type oauthenticator.github.GitHubOAuthenticator + sudo tljh-config set auth.type github ``` -5. Restart your JupyterHub so that new users see these changes: +5. Tell JupyterHub which users to allow, if you haven't already: + + ``` + sudo tljh-config add-item users.allowed good-user_1 + ``` + +6. Restart your JupyterHub so that new users see these changes: ``` sudo tljh-config reload From 634956c33e1444320f56b28c6bf020f72483dfff Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Aug 2024 10:00:57 +0200 Subject: [PATCH 59/61] Update base user environment to miniforge 24.5.0-0 (Python 3.12) mambaforge is deprecated, mamba has been in miniforge for some time now --- docs/topic/installer-actions.md | 2 +- tests/test_conda.py | 4 ++-- tests/test_installer.py | 14 +++++++------- tljh/installer.py | 22 +++++++++++----------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/topic/installer-actions.md b/docs/topic/installer-actions.md index 7dc26ab..b738bf2 100644 --- a/docs/topic/installer-actions.md +++ b/docs/topic/installer-actions.md @@ -26,7 +26,7 @@ sudo rm -rf /opt/tljh/hub ## User environment -By default, a `mambaforge` conda environment is installed in `/opt/tljh/user`. This contains +By default, a `miniforge` conda environment is installed in `/opt/tljh/user`. This contains the notebook interface used to launch all users, and the various packages available to all users. The environment is owned by the `root` user. JupyterHub admins may use to `sudo -E conda install` or `sudo -E pip install` packages into this environment. diff --git a/tests/test_conda.py b/tests/test_conda.py index 397499e..af9b298 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -14,9 +14,9 @@ from tljh import conda, installer @pytest.fixture(scope="module") def prefix(): """ - Provide a temporary directory with a mambaforge conda environment + Provide a temporary directory with a conda environment """ - installer_url, checksum = installer._mambaforge_url() + installer_url, checksum = installer._miniforge_url() with tempfile.TemporaryDirectory() as tmpdir: with conda.download_miniconda_installer( installer_url, checksum diff --git a/tests/test_installer.py b/tests/test_installer.py index f06bb6e..f5c218e 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -46,12 +46,12 @@ def test_ensure_admins(tljh_dir, admins, expected_config): def setup_conda(distro, version, prefix): - """Install mambaforge or miniconda in a prefix""" + """Install miniforge or miniconda in a prefix""" if distro == "mambaforge": - installer_url, _ = installer._mambaforge_url(version) + installer_url, _ = installer._miniforge_url(version) + installer_url = installer_url.replace("Miniforge3", "Mambaforge") elif distro == "miniforge": - installer_url, _ = installer._mambaforge_url(version) - installer_url = installer_url.replace("Mambaforge", "Miniforge3") + installer_url, _ = installer._miniforge_url(version) elif distro == "miniconda": arch = os.uname().machine installer_url = ( @@ -124,9 +124,9 @@ def _specifier(version): None, None, { - "python": "3.10.*", - "conda": "23.1.0", - "mamba": "1.4.1", + "python": "3.12.*", + "conda": "24.5.0", + "mamba": "1.5.8", }, ), # previous install, 1.0 diff --git a/tljh/installer.py b/tljh/installer.py index 5a9d767..48b9ddc 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -136,13 +136,13 @@ def ensure_usergroups(): f.write("Defaults exempt_group = jupyterhub-admins\n") -# Install mambaforge using an installer from +# Install miniforge using an installer from # https://github.com/conda-forge/miniforge/releases -MAMBAFORGE_VERSION = "23.1.0-1" +MINIFORGE_VERSION = "24.5.0-0" # sha256 checksums -MAMBAFORGE_CHECKSUMS = { - "aarch64": "d9d89c9e349369702171008d9ee7c5ce80ed420e5af60bd150a3db4bf674443a", - "x86_64": "cfb16c47dc2d115c8b114280aa605e322173f029fdb847a45348bf4bd23c62ab", +MINIFORGE_CHECKSUMS = { + "aarch64": "848f2d6917c473b1091e31a51241a7626d4dac4b90809a9b2ed937e0cea18d87", + "x86_64": "a754b435830e1c038dada434873ad69a99970a4ea17a68d3bbcade0a37c8c8fb", } # minimum versions of packages @@ -156,22 +156,22 @@ MINIMUM_VERSIONS = { } -def _mambaforge_url(version=MAMBAFORGE_VERSION, arch=None): - """Return (URL, checksum) for mambaforge download for a given version and arch +def _miniforge_url(version=MINIFORGE_VERSION, arch=None): + """Return (URL, checksum) for miniforge download for a given version and arch Default values provided for both version and arch """ if arch is None: arch = os.uname().machine - installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format( + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Miniforge3-{v}-Linux-{arch}.sh".format( v=version, arch=arch, ) # Check system architecture, set appropriate installer checksum - checksum = MAMBAFORGE_CHECKSUMS.get(arch) + checksum = MINIFORGE_CHECKSUMS.get(arch) if not checksum: raise ValueError( - f"Unsupported architecture: {arch}. TLJH only supports {','.join(MAMBAFORGE_CHECKSUMS.keys())}" + f"Unsupported architecture: {arch}. TLJH only supports {','.join(MINIFORGE_CHECKSUMS.keys())}" ) return installer_url, checksum @@ -198,7 +198,7 @@ def ensure_user_environment(user_requirements_txt_file): raise OSError(msg) logger.info("Downloading & setting up user environment...") - installer_url, installer_sha256 = _mambaforge_url() + installer_url, installer_sha256 = _miniforge_url() with conda.download_miniconda_installer( installer_url, installer_sha256 ) as installer_path: From 5172ceb4f90e061a29a9a165375b716b307c7053 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 4 Sep 2024 12:00:43 +0200 Subject: [PATCH 60/61] Update base user environment to miniforge 24.7.1-0 (Python 3.12) --- tests/test_installer.py | 4 ++-- tljh/installer.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_installer.py b/tests/test_installer.py index f5c218e..f8db748 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -125,8 +125,8 @@ def _specifier(version): None, { "python": "3.12.*", - "conda": "24.5.0", - "mamba": "1.5.8", + "conda": "24.7.1", + "mamba": "1.5.9", }, ), # previous install, 1.0 diff --git a/tljh/installer.py b/tljh/installer.py index 48b9ddc..b8460b9 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -138,11 +138,11 @@ def ensure_usergroups(): # Install miniforge using an installer from # https://github.com/conda-forge/miniforge/releases -MINIFORGE_VERSION = "24.5.0-0" +MINIFORGE_VERSION = "24.7.1-0" # sha256 checksums MINIFORGE_CHECKSUMS = { - "aarch64": "848f2d6917c473b1091e31a51241a7626d4dac4b90809a9b2ed937e0cea18d87", - "x86_64": "a754b435830e1c038dada434873ad69a99970a4ea17a68d3bbcade0a37c8c8fb", + "aarch64": "7a3372268b45679584043b4ba1e0318ee5027384a8d330f2d991b14d815d6a6d", + "x86_64": "b64f77042cf8eafd31ced64f9253a74fb85db63545fe167ba5756aea0e8125be", } # minimum versions of packages From 8490ef2949d11b857ab1ac35275904d9f3115859 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 4 Sep 2024 12:21:18 +0200 Subject: [PATCH 61/61] Update lower version bound on filelock, and add comment --- tljh/requirements-hub-env.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 556ad19..25bf123 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -27,4 +27,5 @@ jupyterhub-idle-culler>=1.2.1,<2 # pycurl>=7.45.2,<8 -filelock>=3.13.3,<4 +# filelock is used to help us do atomic operations on config file(s) +filelock>=3.15.4,<4