From bdbb3adbbc1670b5f8f0fa7a4e762c249d6a592c Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:49:07 -0400 Subject: [PATCH 1/4] Add config lockfile --- tljh/config.py | 109 +++++++++++++++++++++------------- tljh/requirements-hub-env.txt | 2 + 2 files changed, 69 insertions(+), 42 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 633cc36..d72b03e 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -178,12 +178,7 @@ def show_config(config_path): """ Pretty print config from given config_path """ - try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} - + config = get_current_config(config_path) yaml.dump(config, sys.stdout) @@ -191,75 +186,105 @@ def set_config_value(config_path, key_path, value, validate=True): """ Set key at key_path in config_path to value """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} - config = set_item_in_config(config, key_path, value) + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = set_item_in_config(config, key_path, value) + validate_config(config, validate) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def unset_config_value(config_path, key_path, validate=True): """ Unset key at key_path in config_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = unset_item_from_config(config, key_path) + validate_config(config, validate) - config = unset_item_from_config(config, key_path) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def add_config_value(config_path, key_path, value, validate=True): """ Add value to list at key_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) try: - with open(config_path) as f: - config = yaml.load(f) - except FileNotFoundError: - config = {} + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = add_item_to_config(config, key_path, value) + validate_config(config, validate) - config = add_item_to_config(config, key_path, value) - validate_config(config, validate) + with open(config_path, "w") as f: + yaml.dump(config, f) - with open(config_path, "w") as f: - yaml.dump(config, f) + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) def remove_config_value(config_path, key_path, value, validate=True): """ Remove value from list at key_path """ - # FIXME: Have a file lock here + from filelock import FileLock, Timeout + + lock_file = f"{config_path}.lock" + lock = FileLock(lock_file) + try: + with lock.acquire(timeout=1): + config = get_current_config(config_path) + config = remove_item_from_config(config, key_path, value) + validate_config(config, validate) + + with open(config_path, "w") as f: + yaml.dump(config, f) + + except Timeout: + print(f"Another instance of tljh-config holds the lock {lock_file}") + exit(1) + + +def get_current_config(config_path): + """ + Retrieve the current config at config_path + """ try: with open(config_path) as f: - config = yaml.load(f) + return yaml.load(f) except FileNotFoundError: - config = {} - - config = remove_item_from_config(config, key_path, value) - validate_config(config, validate) - - with open(config_path, "w") as f: - yaml.dump(config, f) + return {} def check_hub_ready(): + """ + Checks that hub is running. + """ from .configurer import load_config base_url = load_config()["base_url"] diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 62f39f4..556ad19 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -26,3 +26,5 @@ jupyterhub-idle-culler>=1.2.1,<2 # ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # pycurl>=7.45.2,<8 + +filelock>=3.13.3,<4 From 13ed32b499c297f4cf9e7ecd0e3abcd89a0597ef Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:51:51 -0400 Subject: [PATCH 2/4] Add `lockfile` to integration tests --- integration-tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index c086c7f..3ef31e4 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,3 +1,4 @@ +filelock pytest pytest-cov pytest-asyncio From 96ac0d3538b1a21c04b712dd1035e0412619d210 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 6 Apr 2024 11:03:24 -0400 Subject: [PATCH 3/4] Add `filelock` to `dev-requirements` --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 672ad32..f3b24c9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ +filelock packaging pytest pytest-cov From 8490ef2949d11b857ab1ac35275904d9f3115859 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 4 Sep 2024 12:21:18 +0200 Subject: [PATCH 4/4] Update lower version bound on filelock, and add comment --- tljh/requirements-hub-env.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 556ad19..25bf123 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -27,4 +27,5 @@ jupyterhub-idle-culler>=1.2.1,<2 # pycurl>=7.45.2,<8 -filelock>=3.13.3,<4 +# filelock is used to help us do atomic operations on config file(s) +filelock>=3.15.4,<4