diff --git a/.github/integration-test.py b/.github/integration-test.py index cf027fc..7e59669 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -25,14 +25,18 @@ def run_systemd_image(image_name, container_name, bootstrap_pip_spec): Container named container_name will be started. """ cmd = [ - 'docker', 'run', + 'docker', + 'run', '--privileged', - '--mount', 'type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup', + '--mount', + 'type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup', '--detach', - '--name', container_name, + '--name', + container_name, # A bit less than 1GB to ensure TLJH runs on 1GB VMs. # If this is changed all docs references to the required memory must be changed too. - '--memory', '900m', + '--memory', + '900m', ] if bootstrap_pip_spec: @@ -49,51 +53,46 @@ def stop_container(container_name): Stop & remove docker container if it exists. """ try: - subprocess.check_output([ - 'docker', 'inspect', container_name - ], stderr=subprocess.STDOUT) + subprocess.check_output( + ['docker', 'inspect', container_name], stderr=subprocess.STDOUT + ) except subprocess.CalledProcessError: # No such container exists, nothing to do return - subprocess.check_call([ - 'docker', 'rm', '-f', container_name - ]) + subprocess.check_call(['docker', 'rm', '-f', container_name]) def run_container_command(container_name, cmd): """ Run cmd in a running container with a bash shell """ - proc = subprocess.run([ - 'docker', 'exec', - '-t', container_name, - '/bin/bash', '-c', cmd - ], check=True) + proc = subprocess.run( + ['docker', 'exec', '-t', container_name, '/bin/bash', '-c', cmd], check=True + ) def copy_to_container(container_name, src_path, dest_path): """ Copy files from src_path to dest_path inside container_name """ - subprocess.check_call([ - 'docker', 'cp', - src_path, f'{container_name}:{dest_path}' - ]) + subprocess.check_call(['docker', 'cp', src_path, f'{container_name}:{dest_path}']) -def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args): +def run_test( + image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args +): """ Wrapper that sets up tljh with installer_args & runs test_name """ stop_container(test_name) run_systemd_image(image_name, test_name, bootstrap_pip_spec) - source_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), os.pardir) - ) + source_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) copy_to_container(test_name, os.path.join(source_path, 'bootstrap/.'), '/srv/src') - copy_to_container(test_name, os.path.join(source_path, 'integration-tests/'), '/srv/src') + copy_to_container( + test_name, os.path.join(source_path, 'integration-tests/'), '/srv/src' + ) # These logs can be very relevant to debug a container startup failure print(f"--- Start of logs from the container: {test_name}") @@ -103,20 +102,16 @@ def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, ins # Install TLJH from the default branch first to test upgrades if upgrade: run_container_command( - test_name, - 'curl -L https://tljh.jupyter.org/bootstrap.py | python3 -' + test_name, 'curl -L https://tljh.jupyter.org/bootstrap.py | python3 -' ) - run_container_command( - test_name, - f'python3 /srv/src/bootstrap.py {installer_args}' - ) + run_container_command(test_name, f'python3 /srv/src/bootstrap.py {installer_args}') # Install pkgs from requirements in hub's pip, where # the bootstrap script installed the others run_container_command( test_name, - '/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt' + '/opt/tljh/hub/bin/python3 -m pip install -r /srv/src/integration-tests/requirements.txt', ) run_container_command( test_name, @@ -124,8 +119,10 @@ def run_test(image_name, test_name, bootstrap_pip_spec, test_files, upgrade, ins # avoid a flood of logs while still understanding if multiple tests # would fail. '/opt/tljh/hub/bin/python3 -m pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no {}'.format( - ' '.join([os.path.join('/srv/src/integration-tests/', f) for f in test_files]) - ) + ' '.join( + [os.path.join('/srv/src/integration-tests/', f) for f in test_files] + ) + ), ) @@ -133,15 +130,12 @@ def show_logs(container_name): """ Print logs from inside container to stdout """ + run_container_command(container_name, 'journalctl --no-pager') run_container_command( - container_name, - 'journalctl --no-pager' - ) - run_container_command( - container_name, - 'systemctl --no-pager status jupyterhub traefik' + container_name, 'systemctl --no-pager status jupyterhub traefik' ) + def main(): argparser = argparse.ArgumentParser() subparsers = argparser.add_subparsers(dest='action') @@ -153,13 +147,9 @@ def main(): dest="build_args", ) - subparsers.add_parser('stop-container').add_argument( - 'container_name' - ) + subparsers.add_parser('stop-container').add_argument('container_name') - subparsers.add_parser('start-container').add_argument( - 'container_name' - ) + subparsers.add_parser('start-container').add_argument('container_name') run_parser = subparsers.add_parser('run') run_parser.add_argument('container_name') @@ -173,7 +163,9 @@ def main(): run_test_parser = subparsers.add_parser('run-test') run_test_parser.add_argument('--installer-args', default='') run_test_parser.add_argument('--upgrade', action='store_true') - run_test_parser.add_argument('--bootstrap-pip-spec', nargs='?', default="", type=str) + run_test_parser.add_argument( + '--bootstrap-pip-spec', nargs='?', default="", type=str + ) run_test_parser.add_argument('test_name') run_test_parser.add_argument('test_files', nargs='+') @@ -185,7 +177,14 @@ def main(): image_name = 'tljh-systemd' if args.action == 'run-test': - run_test(image_name, args.test_name, args.bootstrap_pip_spec, args.test_files, args.upgrade, args.installer_args) + run_test( + image_name, + args.test_name, + args.bootstrap_pip_spec, + args.test_files, + args.upgrade, + args.installer_args, + ) elif args.action == 'show-logs': show_logs(args.container_name) elif args.action == 'run': diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 0315c13..14d0f67 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -145,20 +145,26 @@ def run_subprocess(cmd, *args, **kwargs): and failed output directly to the user's screen """ logger = logging.getLogger('tljh') - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs) + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs + ) printable_command = ' '.join(cmd) if proc.returncode != 0: # Our process failed! Show output to the user - logger.error('Ran {command} with exit code {code}'.format( - command=printable_command, code=proc.returncode - )) + logger.error( + 'Ran {command} with exit code {code}'.format( + command=printable_command, code=proc.returncode + ) + ) logger.error(proc.stdout.decode()) raise subprocess.CalledProcessError(cmd=cmd, returncode=proc.returncode) else: # This goes into installer.log - logger.debug('Ran {command} with exit code {code}'.format( - command=printable_command, code=proc.returncode - )) + logger.debug( + 'Ran {command} with exit code {code}'.format( + command=printable_command, code=proc.returncode + ) + ) # This produces multi line log output, unfortunately. Not sure how to fix. # For now, prioritizing human readability over machine readability. logger.debug(proc.stdout.decode()) @@ -169,6 +175,7 @@ def ensure_host_system_can_install_tljh(): Check if TLJH is installable in current host system and exit with a clear error message otherwise. """ + def get_os_release_variable(key): """ Return value for key from /etc/os-release @@ -177,10 +184,17 @@ def ensure_host_system_can_install_tljh(): Returns empty string if key is not found. """ - return subprocess.check_output([ - '/bin/bash', '-c', - "source /etc/os-release && echo ${{{key}}}".format(key=key) - ]).decode().strip() + return ( + subprocess.check_output( + [ + '/bin/bash', + '-c', + "source /etc/os-release && echo ${{{key}}}".format(key=key), + ] + ) + .decode() + .strip() + ) # Require Ubuntu 18.04+ distro = get_os_release_variable('ID') @@ -203,8 +217,12 @@ def ensure_host_system_can_install_tljh(): # Provide additional information about running in docker containers if os.path.exists('/.dockerenv'): print("Running inside a docker container without systemd isn't supported") - print("We recommend against running a production TLJH instance inside a docker container") - print("For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html") + print( + "We recommend against running a production TLJH instance inside a docker container" + ) + print( + "For local development, see http://tljh.jupyter.org/en/latest/contributing/dev-setup.html" + ) sys.exit(1) @@ -226,7 +244,7 @@ class ProgressPageRequestHandler(SimpleHTTPRequestHandler): return SimpleHTTPRequestHandler.do_GET(self) elif self.path == "/": self.send_response(302) - self.send_header('Location','/index.html') + self.send_header('Location', '/index.html') self.end_headers() else: SimpleHTTPRequestHandler.send_error(self, code=403) @@ -243,7 +261,6 @@ def main(): """ ensure_host_system_can_install_tljh() - # Various related constants install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh') hub_prefix = os.path.join(install_prefix, 'hub') @@ -251,7 +268,6 @@ def main(): pip_bin = os.path.join(hub_prefix, 'bin', 'pip') initial_setup = not os.path.exists(python_bin) - # Attempt to start a web server to serve a progress page reporting # installation progress. tljh_installer_flags = sys.argv[1:] @@ -276,8 +292,11 @@ def main(): server.serve_forever() except KeyboardInterrupt: pass + progress_page_server = HTTPServer(("", 80), ProgressPageRequestHandler) - p = multiprocessing.Process(target=serve_forever, args=(progress_page_server,)) + p = multiprocessing.Process( + target=serve_forever, args=(progress_page_server,) + ) p.start() # Pass the server's pid to the installer for later termination @@ -285,7 +304,6 @@ def main(): except OSError: pass - # Set up logging to print to a file and to stderr os.makedirs(install_prefix, exist_ok=True) file_logger_path = os.path.join(install_prefix, 'installer.log') @@ -304,7 +322,6 @@ def main(): logger.setLevel(logging.DEBUG) - if not initial_setup: logger.info('Existing TLJH installation detected, upgrading...') else: @@ -324,23 +341,35 @@ def main(): apt_get_adjusted_env = os.environ.copy() apt_get_adjusted_env["DEBIAN_FRONTEND"] = "noninteractive" run_subprocess(['apt-get', 'update']) - run_subprocess(['apt-get', 'install', '--yes', 'software-properties-common'], env=apt_get_adjusted_env) + run_subprocess( + ['apt-get', 'install', '--yes', 'software-properties-common'], + env=apt_get_adjusted_env, + ) run_subprocess(['add-apt-repository', 'universe', '--yes']) run_subprocess(['apt-get', 'update']) - run_subprocess(['apt-get', 'install', '--yes', 'python3', 'python3-venv', 'python3-pip', 'git'], env=apt_get_adjusted_env) + run_subprocess( + [ + 'apt-get', + 'install', + '--yes', + 'python3', + 'python3-venv', + 'python3-pip', + 'git', + ], + env=apt_get_adjusted_env, + ) logger.info('Setting up virtual environment at {}'.format(hub_prefix)) os.makedirs(hub_prefix, exist_ok=True) run_subprocess(['python3', '-m', 'venv', hub_prefix]) - # Upgrade pip # Keep pip version pinning in sync with the one in unit-test.yml! # See changelog at https://pip.pypa.io/en/latest/news/#changelog logger.info('Upgrading pip...') run_subprocess([pip_bin, 'install', '--upgrade', 'pip==21.3.*']) - # Install/upgrade TLJH installer tljh_install_cmd = [pip_bin, 'install', '--upgrade'] if os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes': @@ -348,7 +377,7 @@ def main(): tljh_install_cmd.append( os.environ.get( 'TLJH_BOOTSTRAP_PIP_SPEC', - 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git' + 'git+https://github.com/jupyterhub/the-littlest-jupyterhub.git', ) ) if initial_setup: @@ -357,7 +386,6 @@ def main(): logger.info('Upgrading TLJH installer...') run_subprocess(tljh_install_cmd) - # Run TLJH installer logger.info('Running TLJH installer...') os.execv(python_bin, [python_bin, '-m', 'tljh.installer'] + tljh_installer_flags) diff --git a/docs/conf.py b/docs/conf.py index 2558b8b..b3c283f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,7 @@ version = '' release = 'v0.1' # Enable MathJax for Math -extensions = ['sphinx.ext.mathjax', - 'sphinx.ext.intersphinx', - 'sphinx_copybutton'] +extensions = ['sphinx.ext.mathjax', 'sphinx.ext.intersphinx', 'sphinx_copybutton'] # The root toctree document. root_doc = master_doc = "index" @@ -22,14 +20,13 @@ root_doc = master_doc = "index" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', - 'install/custom.rst'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'install/custom.rst'] intersphinx_mapping = { 'sphinx': ('http://www.sphinx-doc.org/en/master/', None), } -intersphinx_cache_limit = 90 # days +intersphinx_cache_limit = 90 # days # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 82dd70e..123be8d 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -5,13 +5,13 @@ import os from pytest import fixture - @fixture def preserve_config(request): """Fixture to save and restore config around tests""" # Import TLJH only when needed. This lets us run tests in places # where TLJH is not installed - particularly, the 'distro check' test. from tljh.config import CONFIG_FILE, reload_component + if os.path.exists(CONFIG_FILE): with open(CONFIG_FILE) as f: save_config = f.read() diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index d9176de..7c287dd 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -17,12 +17,14 @@ def tljh_extra_user_pip_packages(): 'django', ] + @hookimpl def tljh_extra_hub_pip_packages(): return [ 'there', ] + @hookimpl def tljh_extra_apt_packages(): return [ @@ -33,9 +35,8 @@ def tljh_extra_apt_packages(): @hookimpl def tljh_config_post_install(config): # Put an arbitrary marker we can test for - config['simplest_plugin'] = { - 'present': True - } + config['simplest_plugin'] = {'present': True} + @hookimpl def tljh_custom_jupyterhub_config(c): diff --git a/integration-tests/test_admin_installer.py b/integration-tests/test_admin_installer.py index 1bda5d4..f710911 100644 --- a/integration-tests/test_admin_installer.py +++ b/integration-tests/test_admin_installer.py @@ -19,6 +19,7 @@ async def test_admin_login(): # If user is not logged in, this will raise an exception await u.ensure_server_simulate() + @pytest.mark.asyncio @pytest.mark.parametrize( "username, password", diff --git a/integration-tests/test_bootstrap.py b/integration-tests/test_bootstrap.py index de95a46..b0cdab6 100644 --- a/integration-tests/test_bootstrap.py +++ b/integration-tests/test_bootstrap.py @@ -38,6 +38,7 @@ def get_bootstrap_script_location(container_name, show_progress_page): subprocess.check_call(["docker", "cp", source_path, f"{container_name}:/srv/src"]) return bootstrap_script + # FIXME: Refactor this function to easier to understand using the following # parameters # @@ -55,7 +56,9 @@ def get_bootstrap_script_location(container_name, show_progress_page): # running against the systemd container that cab be built by # integration-test.py. # -def run_bootstrap_after_preparing_container(container_name, image, show_progress_page=False): +def run_bootstrap_after_preparing_container( + container_name, image, show_progress_page=False +): """ 1. Stops old container 2. Starts --detached container @@ -163,7 +166,7 @@ def test_progress_page(): run_bootstrap_after_preparing_container, "progress-page", f"ubuntu:{os.getenv('UBUNTU_VERSION', '20.04')}", - True + True, ) # Check if progress page started diff --git a/integration-tests/test_extensions.py b/integration-tests/test_extensions.py index 13b305a..776da68 100644 --- a/integration-tests/test_extensions.py +++ b/integration-tests/test_extensions.py @@ -6,39 +6,43 @@ def test_serverextensions(): Validate serverextensions we want are installed """ # jupyter-serverextension writes to stdout and stderr weirdly - proc = subprocess.run([ - '/opt/tljh/user/bin/jupyter-serverextension', - 'list', '--sys-prefix' - ], stderr=subprocess.PIPE) + proc = subprocess.run( + ['/opt/tljh/user/bin/jupyter-serverextension', 'list', '--sys-prefix'], + stderr=subprocess.PIPE, + ) extensions = [ 'jupyterlab 3.', 'nbgitpuller 1.', 'nteract_on_jupyter 2.1.', - 'jupyter_resource_usage' + 'jupyter_resource_usage', ] for e in extensions: assert e in proc.stderr.decode() + def test_nbextensions(): """ Validate nbextensions we want are installed & enabled """ # jupyter-nbextension writes to stdout and stderr weirdly - proc = subprocess.run([ - '/opt/tljh/user/bin/jupyter-nbextension', - 'list', '--sys-prefix' - ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + proc = subprocess.run( + ['/opt/tljh/user/bin/jupyter-nbextension', 'list', '--sys-prefix'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) extensions = [ 'jupyter_resource_usage/main', # This is what ipywidgets nbextension is called - 'jupyter-js-widgets/extension' + 'jupyter-js-widgets/extension', ] for e in extensions: assert f'{e} \x1b[32m enabled \x1b[0m' in proc.stdout.decode() # Ensure we have 'OK' messages in our stdout, to make sure everything is importable - assert proc.stderr.decode() == ' - Validating: \x1b[32mOK\x1b[0m\n' * len(extensions) + assert proc.stderr.decode() == ' - Validating: \x1b[32mOK\x1b[0m\n' * len( + extensions + ) diff --git a/integration-tests/test_hub.py b/integration-tests/test_hub.py index 5e98d1a..9d56f43 100644 --- a/integration-tests/test_hub.py +++ b/integration-tests/test_hub.py @@ -33,8 +33,20 @@ async def test_user_code_execute(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() @@ -57,17 +69,48 @@ async def test_user_server_started_with_custom_base_url(): hub_url = f"http://localhost{base_url}" username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'base_url', base_url + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() await u.ensure_server_simulate() # unset base_url to avoid problems with other tests - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'unset', 'base_url')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'unset', 'base_url' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) @pytest.mark.asyncio @@ -80,9 +123,28 @@ async def test_user_admin_add(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'add-item', 'users.admin', username + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() @@ -92,8 +154,7 @@ async def test_user_admin_add(): assert pwd.getpwnam(f'jupyter-{username}') is not None # Assert that the user has admin rights - assert f'jupyter-{username}' in grp.getgrnam( - 'jupyterhub-admins').gr_mem + assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem # FIXME: Make this test pass @@ -110,9 +171,28 @@ async def test_user_admin_remove(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'add-item', 'users.admin', username + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() @@ -122,18 +202,28 @@ async def test_user_admin_remove(): assert pwd.getpwnam(f'jupyter-{username}') is not None # Assert that the user has admin rights - assert f'jupyter-{username}' in grp.getgrnam( - 'jupyterhub-admins').gr_mem + assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) await u.stop_server() await u.ensure_server_simulate() # Assert that the user does *not* have admin rights - assert f'jupyter-{username}' not in grp.getgrnam( - 'jupyterhub-admins').gr_mem + assert f'jupyter-{username}' not in grp.getgrnam('jupyterhub-admins').gr_mem @pytest.mark.asyncio @@ -146,8 +236,20 @@ async def test_long_username(): hub_url = 'http://localhost' username = secrets.token_hex(32) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) try: async with User(username, hub_url, partial(login_dummy, password='')) as u: @@ -161,11 +263,7 @@ async def test_long_username(): await u.stop_server() except: # If we have any errors, print jupyterhub logs before exiting - subprocess.check_call([ - 'journalctl', - '-u', 'jupyterhub', - '--no-pager' - ]) + subprocess.check_call(['journalctl', '-u', 'jupyterhub', '--no-pager']) raise @@ -182,9 +280,31 @@ async def test_user_group_adding(): # Create the group we want to add the user to system('groupadd somegroup') - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, + 'add-item', + 'users.extra_user_groups.somegroup', + username, + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) try: async with User(username, hub_url, partial(login_dummy, password='')) as u: @@ -203,11 +323,7 @@ async def test_user_group_adding(): system('groupdel somegroup') except: # If we have any errors, print jupyterhub logs before exiting - subprocess.check_call([ - 'journalctl', - '-u', 'jupyterhub', - '--no-pager' - ]) + subprocess.check_call(['journalctl', '-u', 'jupyterhub', '--no-pager']) raise @@ -222,14 +338,47 @@ async def test_idle_server_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) # Check every 10s for idle servers to cull - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10" + ) + ).wait() + ) # Apart from servers, also cull users - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True" + ) + ).wait() + ) # Cull servers and users after 60s of activity - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60" + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() @@ -239,14 +388,18 @@ async def test_idle_server_culled(): assert pwd.getpwnam(f'jupyter-{username}') is not None # Check that we can get to the user's server - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) + r = await u.session.get( + u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}, + ) assert r.status == 200 async def _check_culling_done(): # Check that after 60s, the user and server have been culled and are not reacheable anymore - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) + r = await u.session.get( + u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}, + ) print(r.status) return r.status == 403 @@ -268,14 +421,47 @@ async def test_active_server_not_culled(): hub_url = 'http://localhost' username = secrets.token_hex(8) - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy' + ) + ).wait() + ) # Check every 10s for idle servers to cull - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10" + ) + ).wait() + ) # Apart from servers, also cull users - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True")).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.users', "True" + ) + ).wait() + ) # Cull servers and users after 60s of activity - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60")).wait() - assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait() + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec( + *TLJH_CONFIG_PATH, 'set', 'services.cull.max_age', "60" + ) + ).wait() + ) + assert ( + 0 + == await ( + await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload') + ).wait() + ) async with User(username, hub_url, partial(login_dummy, password='')) as u: await u.login() @@ -285,14 +471,18 @@ async def test_active_server_not_culled(): assert pwd.getpwnam(f'jupyter-{username}') is not None # Check that we can get to the user's server - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) + r = await u.session.get( + u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}, + ) assert r.status == 200 async def _check_culling_done(): # Check that after 30s, we can still reach the user's server - r = await u.session.get(u.hub_url / 'hub/api/users' / username, - headers={'Referer': str(u.hub_url / 'hub/')}) + r = await u.session.get( + u.hub_url / 'hub/api/users' / username, + headers={'Referer': str(u.hub_url / 'hub/')}, + ) print(r.status) return r.status != 200 diff --git a/integration-tests/test_install.py b/integration-tests/test_install.py index 58f2815..8d2b06d 100644 --- a/integration-tests/test_install.py +++ b/integration-tests/test_install.py @@ -172,7 +172,16 @@ def test_pip_install(group, allowed): # get a failure if the user can't install to the global site. Which is # what we wanted to test for here. subprocess.check_call( - [python, "-m", "pip", "install", "--no-user", "--ignore-installed", "--no-deps", "flit"], + [ + python, + "-m", + "pip", + "install", + "--no-user", + "--ignore-installed", + "--no-deps", + "flit", + ], preexec_fn=partial(setgroup, group), ) if allowed: diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index 63e4841..2cb78f7 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -90,7 +90,6 @@ def test_manual_https(preserve_config): ) assert resp.code == 200 - # cleanup shutil.rmtree(ssl_dir) set_config_value(CONFIG_FILE, "https.enabled", False) @@ -105,7 +104,6 @@ def test_extra_traefik_config(): dynamic_config_dir = os.path.join(STATE_DIR, "rules") os.makedirs(dynamic_config_dir, exist_ok=True) - extra_static_config = { "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}}, "api": {"dashboard": True, "entrypoint": "no_auth_api"}, @@ -126,7 +124,6 @@ def test_extra_traefik_config(): }, } - success = False for i in range(5): time.sleep(i) diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index e61bbcc..7888369 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -23,28 +23,16 @@ def test_pip_packages(): """ Test extra user & hub pip packages are installed """ - subprocess.check_call([ - f'{USER_ENV_PREFIX}/bin/python3', - '-c', - 'import django' - ]) + subprocess.check_call([f'{USER_ENV_PREFIX}/bin/python3', '-c', 'import django']) - subprocess.check_call([ - f'{HUB_ENV_PREFIX}/bin/python3', - '-c', - 'import there' - ]) + subprocess.check_call([f'{HUB_ENV_PREFIX}/bin/python3', '-c', 'import there']) def test_conda_packages(): """ Test extra user conda packages are installed """ - subprocess.check_call([ - f'{USER_ENV_PREFIX}/bin/python3', - '-c', - 'import hypothesis' - ]) + subprocess.check_call([f'{USER_ENV_PREFIX}/bin/python3', '-c', 'import hypothesis']) def test_config_hook(): @@ -80,7 +68,7 @@ def test_new_user_create(): """ Test that plugin receives username as arg """ - username="user1" + username = "user1" # Call ensure_user to make sure the user plugin gets called user.ensure_user(username) diff --git a/tests/test_conda.py b/tests/test_conda.py index 7632673..19ccd1a 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -16,16 +16,22 @@ def prefix(): # see https://github.com/conda-forge/miniforge/releases mambaforge_version = '4.10.3-7' if os.uname().machine == 'aarch64': - installer_sha256 = "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2" + installer_sha256 = ( + "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2" + ) elif os.uname().machine == 'x86_64': - installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" - installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(v=mambaforge_version, arch=os.uname().machine) + installer_sha256 = ( + "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" + ) + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format( + v=mambaforge_version, arch=os.uname().machine + ) with tempfile.TemporaryDirectory() as tmpdir: - with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: + with conda.download_miniconda_installer( + installer_url, installer_sha256 + ) as installer_path: conda.install_miniconda(installer_path, tmpdir) - conda.ensure_conda_packages(tmpdir, [ - 'conda==4.10.3' - ]) + conda.ensure_conda_packages(tmpdir, ['conda==4.10.3']) yield tmpdir @@ -35,11 +41,7 @@ def test_ensure_packages(prefix): """ conda.ensure_conda_packages(prefix, ['numpy']) # Throws an error if this fails - subprocess.check_call([ - os.path.join(prefix, 'bin', 'python'), - '-c', - 'import numpy' - ]) + subprocess.check_call([os.path.join(prefix, 'bin', 'python'), '-c', 'import numpy']) def test_ensure_pip_packages(prefix): @@ -49,11 +51,7 @@ def test_ensure_pip_packages(prefix): conda.ensure_conda_packages(prefix, ['pip']) conda.ensure_pip_packages(prefix, ['numpy']) # Throws an error if this fails - subprocess.check_call([ - os.path.join(prefix, 'bin', 'python'), - '-c', - 'import numpy' - ]) + subprocess.check_call([os.path.join(prefix, 'bin', 'python'), '-c', 'import numpy']) def test_ensure_pip_requirements(prefix): @@ -66,8 +64,4 @@ def test_ensure_pip_requirements(prefix): f.write(b'there') f.flush() conda.ensure_pip_requirements(prefix, f.name) - subprocess.check_call([ - os.path.join(prefix, 'bin', 'python'), - '-c', - 'import there' - ]) + subprocess.check_call([os.path.join(prefix, 'bin', 'python'), '-c', 'import there']) diff --git a/tests/test_config.py b/tests/test_config.py index d7bba6c..0f63d25 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -32,10 +32,7 @@ def test_set_multi_level(): new_conf = config.set_item_in_config(conf, 'a.b', 'c') new_conf = config.set_item_in_config(new_conf, 'a.d', 'e') new_conf = config.set_item_in_config(new_conf, 'f', 'g') - assert new_conf == { - 'a': {'b': 'c', 'd': 'e'}, - 'f': 'g' - } + assert new_conf == {'a': {'b': 'c', 'd': 'e'}, 'f': 'g'} def test_set_overwrite(): @@ -44,9 +41,7 @@ def test_set_overwrite(): This might be surprising destructive behavior to some :D """ - conf = { - 'a': 'b' - } + conf = {'a': 'b'} new_conf = config.set_item_in_config(conf, 'a', 'c') assert new_conf == {'a': 'c'} @@ -73,16 +68,10 @@ def test_unset_one_level(): def test_unset_multi_level(): - conf = { - 'a': {'b': 'c', 'd': 'e'}, - 'f': 'g' - } + conf = {'a': {'b': 'c', 'd': 'e'}, 'f': 'g'} new_conf = config.unset_item_from_config(conf, 'a.b') - assert new_conf == { - 'a': {'d': 'e'}, - 'f': 'g' - } + assert new_conf == {'a': {'d': 'e'}, 'f': 'g'} new_conf = config.unset_item_from_config(new_conf, 'a.d') assert new_conf == {'f': 'g'} new_conf = config.unset_item_from_config(new_conf, 'f') @@ -90,9 +79,7 @@ def test_unset_multi_level(): def test_unset_and_clean_empty_configs(): - conf = { - 'a': {'b': {'c': {'d': {'e': 'f'}}}} - } + conf = {'a': {'b': {'c': {'d': {'e': 'f'}}}}} new_conf = config.unset_item_from_config(conf, 'a.b.c.d.e') assert new_conf == {} @@ -113,32 +100,24 @@ def test_add_to_config_one_level(): conf = {} new_conf = config.add_item_to_config(conf, 'a.b', 'c') - assert new_conf == { - 'a': {'b': ['c']} - } + assert new_conf == {'a': {'b': ['c']}} def test_add_to_config_zero_level(): conf = {} new_conf = config.add_item_to_config(conf, 'a', 'b') - assert new_conf == { - 'a': ['b'] - } + assert new_conf == {'a': ['b']} def test_add_to_config_multiple(): conf = {} new_conf = config.add_item_to_config(conf, 'a.b.c', 'd') - assert new_conf == { - 'a': {'b': {'c': ['d']}} - } + assert new_conf == {'a': {'b': {'c': ['d']}}} new_conf = config.add_item_to_config(new_conf, 'a.b.c', 'e') - assert new_conf == { - 'a': {'b': {'c': ['d', 'e']}} - } + assert new_conf == {'a': {'b': {'c': ['d', 'e']}}} def test_remove_from_config(): @@ -146,14 +125,10 @@ def test_remove_from_config(): new_conf = config.add_item_to_config(conf, 'a.b.c', 'd') new_conf = config.add_item_to_config(new_conf, 'a.b.c', 'e') - assert new_conf == { - 'a': {'b': {'c': ['d', 'e']}} - } + assert new_conf == {'a': {'b': {'c': ['d', 'e']}}} new_conf = config.remove_item_from_config(new_conf, 'a.b.c', 'e') - assert new_conf == { - 'a': {'b': {'c': ['d']}} - } + assert new_conf == {'a': {'b': {'c': ['d']}}} def test_remove_from_config_error(): @@ -193,13 +168,7 @@ def test_cli_no_command(capsys): assert "positional arguments:" in captured.out -@pytest.mark.parametrize( - "arg, value", - [ - ("true", True), - ("FALSE", False) - ] -) +@pytest.mark.parametrize("arg, value", [("true", True), ("FALSE", False)]) def test_cli_set_bool(tljh_dir, arg, value): config.main(["set", "https.enabled", arg]) cfg = configurer.load_config() diff --git a/tests/test_configurer.py b/tests/test_configurer.py index 8865687..432c86f 100644 --- a/tests/test_configurer.py +++ b/tests/test_configurer.py @@ -9,9 +9,6 @@ import sys from tljh import configurer - - - def apply_mock_config(overrides): """ Configure a mock configurer with given overrides. @@ -86,7 +83,10 @@ def test_auth_default(): """ c = apply_mock_config({}) - assert c.JupyterHub.authenticator_class == 'firstuseauthenticator.FirstUseAuthenticator' + assert ( + c.JupyterHub.authenticator_class + == 'firstuseauthenticator.FirstUseAuthenticator' + ) # Do not auto create users who haven't been manually added by default assert not c.FirstUseAuthenticator.create_users @@ -95,14 +95,9 @@ def test_auth_dummy(): """ Test setting Dummy Authenticator & password """ - c = apply_mock_config({ - 'auth': { - 'type': 'dummy', - 'DummyAuthenticator': { - 'password': 'test' - } - } - }) + c = apply_mock_config( + {'auth': {'type': 'dummy', 'DummyAuthenticator': {'password': 'test'}}} + ) assert c.JupyterHub.authenticator_class == 'dummy' assert c.DummyAuthenticator.password == 'test' @@ -111,33 +106,32 @@ def test_user_groups(): """ Test setting user groups """ - c = apply_mock_config({ - 'users': { - 'extra_user_groups': { - "g1": ["u1", "u2"], - "g2": ["u3", "u4"] - }, + c = apply_mock_config( + { + 'users': { + 'extra_user_groups': {"g1": ["u1", "u2"], "g2": ["u3", "u4"]}, + } } - }) - assert c.UserCreatingSpawner.user_groups == { - "g1": ["u1", "u2"], - "g2": ["u3", "u4"] - } + ) + assert c.UserCreatingSpawner.user_groups == {"g1": ["u1", "u2"], "g2": ["u3", "u4"]} def test_auth_firstuse(): """ Test setting FirstUse Authenticator options """ - c = apply_mock_config({ - 'auth': { - 'type': 'firstuseauthenticator.FirstUseAuthenticator', - 'FirstUseAuthenticator': { - 'create_users': True + c = apply_mock_config( + { + 'auth': { + 'type': 'firstuseauthenticator.FirstUseAuthenticator', + 'FirstUseAuthenticator': {'create_users': True}, } } - }) - assert c.JupyterHub.authenticator_class == 'firstuseauthenticator.FirstUseAuthenticator' + ) + assert ( + c.JupyterHub.authenticator_class + == 'firstuseauthenticator.FirstUseAuthenticator' + ) assert c.FirstUseAuthenticator.create_users @@ -145,16 +139,20 @@ def test_auth_github(): """ Test using GitHub authenticator """ - c = apply_mock_config({ - 'auth': { - 'type': 'oauthenticator.github.GitHubOAuthenticator', - 'GitHubOAuthenticator': { - 'client_id': 'something', - 'client_secret': 'something-else' + c = apply_mock_config( + { + 'auth': { + 'type': 'oauthenticator.github.GitHubOAuthenticator', + 'GitHubOAuthenticator': { + 'client_id': 'something', + 'client_secret': 'something-else', + }, } } - }) - assert c.JupyterHub.authenticator_class == 'oauthenticator.github.GitHubOAuthenticator' + ) + assert ( + c.JupyterHub.authenticator_class == 'oauthenticator.github.GitHubOAuthenticator' + ) assert c.GitHubOAuthenticator.client_id == 'something' assert c.GitHubOAuthenticator.client_secret == 'something-else' @@ -173,12 +171,9 @@ def test_set_traefik_api(): """ Test setting per traefik api credentials """ - c = apply_mock_config({ - 'traefik_api': { - 'username': 'some_user', - 'password': '1234' - } - }) + c = apply_mock_config( + {'traefik_api': {'username': 'some_user', 'password': '1234'}} + ) assert c.TraefikTomlProxy.traefik_api_username == 'some_user' assert c.TraefikTomlProxy.traefik_api_password == '1234' @@ -190,40 +185,47 @@ def test_cull_service_default(): c = apply_mock_config({}) cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler', - '--timeout=600', '--cull-every=60', '--concurrency=5', - '--max-age=0' + sys.executable, + '-m', + 'jupyterhub_idle_culler', + '--timeout=600', + '--cull-every=60', + '--concurrency=5', + '--max-age=0', + ] + assert c.JupyterHub.services == [ + { + 'name': 'cull-idle', + 'admin': True, + 'command': cull_cmd, + } ] - assert c.JupyterHub.services == [{ - 'name': 'cull-idle', - 'admin': True, - 'command': cull_cmd, - }] def test_set_cull_service(): """ Test setting cull service options """ - c = apply_mock_config({ - 'services': { - 'cull': { - 'every': 10, - 'users': True, - 'max_age': 60 - } - } - }) + c = apply_mock_config( + {'services': {'cull': {'every': 10, 'users': True, 'max_age': 60}}} + ) cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler', - '--timeout=600', '--cull-every=10', '--concurrency=5', - '--max-age=60', '--cull-users' + sys.executable, + '-m', + 'jupyterhub_idle_culler', + '--timeout=600', + '--cull-every=10', + '--concurrency=5', + '--max-age=60', + '--cull-users', + ] + assert c.JupyterHub.services == [ + { + 'name': 'cull-idle', + 'admin': True, + 'command': cull_cmd, + } ] - assert c.JupyterHub.services == [{ - 'name': 'cull-idle', - 'admin': True, - 'command': cull_cmd, - }] def test_load_secrets(tljh_dir): @@ -243,13 +245,15 @@ def test_auth_native(): """ Test setting Native Authenticator """ - c = apply_mock_config({ - 'auth': { - 'type': 'nativeauthenticator.NativeAuthenticator', - 'NativeAuthenticator': { - 'open_signup': True, + c = apply_mock_config( + { + 'auth': { + 'type': 'nativeauthenticator.NativeAuthenticator', + 'NativeAuthenticator': { + 'open_signup': True, + }, } } - }) + ) assert c.JupyterHub.authenticator_class == 'nativeauthenticator.NativeAuthenticator' assert c.NativeAuthenticator.open_signup == True diff --git a/tests/test_installer.py b/tests/test_installer.py index 361d394..dd4fc1a 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -19,20 +19,20 @@ def test_ensure_config_yaml(tljh_dir): @pytest.mark.parametrize( - "admins, expected_config", - [ - ([['a1'], ['a2'], ['a3']], ['a1', 'a2', 'a3']), - ([['a1:p1'], ['a2']], ['a1', 'a2']), - ], + "admins, expected_config", + [ + ([['a1'], ['a2'], ['a3']], ['a1', 'a2', 'a3']), + ([['a1:p1'], ['a2']], ['a1', 'a2']), + ], ) def test_ensure_admins(tljh_dir, admins, expected_config): - # --admin option called multiple times on the installer - # creates a list of argument lists. - installer.ensure_admins(admins) + # --admin option called multiple times on the installer + # creates a list of argument lists. + installer.ensure_admins(admins) - config_path = installer.CONFIG_FILE - with open(config_path) as f: - config = yaml.load(f) + config_path = installer.CONFIG_FILE + with open(config_path) as f: + config = yaml.load(f) - # verify the list was flattened - assert config['users']['admin'] == expected_config + # verify the list was flattened + assert config['users']['admin'] == expected_config diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 9676365..175bdd6 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -17,8 +17,7 @@ def test_generate_username(): 'jupyter-abcdefghijklmnopq': 'jupyter-abcdefghijklmnopq', # 27 characters, just above our cutoff for hashing 'jupyter-abcdefghijklmnopqr': 'jupyter-abcdefghijklmnopqr-e375e', - } for hub_user, system_user in usernames.items(): assert generate_system_username(hub_user) == system_user - assert len(system_user) <= 32 \ No newline at end of file + assert len(system_user) <= 32 diff --git a/tests/test_traefik.py b/tests/test_traefik.py index e858e07..ecda0ce 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -36,10 +36,8 @@ def test_default_config(tmpdir, tljh_dir): "http": {"address": ":80"}, "auth_api": { "address": "127.0.0.1:8099", - "auth": { - "basic": {"users": [""]} - }, - "whiteList": {"sourceRange": ["127.0.0.1"]} + "auth": {"basic": {"users": [""]}}, + "whiteList": {"sourceRange": ["127.0.0.1"]}, }, } @@ -72,10 +70,8 @@ def test_letsencrypt_config(tljh_dir): "https": {"address": ":443", "tls": {"minVersion": "VersionTLS12"}}, "auth_api": { "address": "127.0.0.1:8099", - "auth": { - "basic": {"users": [""]} - }, - "whiteList": {"sourceRange": ["127.0.0.1"]} + "auth": {"basic": {"users": [""]}}, + "whiteList": {"sourceRange": ["127.0.0.1"]}, }, } assert cfg["acme"] == { @@ -113,18 +109,17 @@ def test_manual_ssl_config(tljh_dir): "minVersion": "VersionTLS12", "certificates": [ {"certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key"} - ] + ], }, }, "auth_api": { "address": "127.0.0.1:8099", - "auth": { - "basic": {"users": [""]} - }, - "whiteList": {"sourceRange": ["127.0.0.1"]} + "auth": {"basic": {"users": [""]}}, + "whiteList": {"sourceRange": ["127.0.0.1"]}, }, } + def test_extra_config(tmpdir, tljh_dir): extra_config_dir = os.path.join(tljh_dir, config.CONFIG_DIR, "traefik_config.d") state_dir = tmpdir.mkdir("state") @@ -146,13 +141,9 @@ def test_extra_config(tmpdir, tljh_dir): # modify existing value "logLevel": "ERROR", # modify existing value with multiple levels - "entryPoints": { - "auth_api": { - "address": "127.0.0.1:9999" - } - }, + "entryPoints": {"auth_api": {"address": "127.0.0.1:9999"}}, # add new setting - "checkNewVersion": False + "checkNewVersion": False, } with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file: diff --git a/tests/test_user.py b/tests/test_user.py index 0cc079a..dc98a12 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -31,9 +31,15 @@ def test_ensure_user(): assert os.path.exists(home_dir) # Ensure not word readable/writable especially in teaching context homedir_stats = os.stat(home_dir).st_mode - assert not (homedir_stats & stat.S_IROTH), "Everyone should not be able to read users home directory" - assert not (homedir_stats & stat.S_IWOTH), "Everyone should not be able to write users home directory" - assert not (homedir_stats & stat.S_IXOTH), "Everyone should not be able to list what is in users home directory" + assert not ( + homedir_stats & stat.S_IROTH + ), "Everyone should not be able to read users home directory" + assert not ( + homedir_stats & stat.S_IWOTH + ), "Everyone should not be able to write users home directory" + assert not ( + homedir_stats & stat.S_IXOTH + ), "Everyone should not be able to list what is in users home directory" # Run ensure_user again, should be a noop user.ensure_user(username) diff --git a/tests/test_utils.py b/tests/test_utils.py index b123c95..555e9f2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,9 +8,7 @@ def test_run_subprocess_exception(mocker): logger = logging.getLogger('tljh') mocker.patch.object(logger, 'error') with pytest.raises(subprocess.CalledProcessError): - utils.run_subprocess( - ['/bin/bash', '-c', 'echo error; exit 1'] - ) + utils.run_subprocess(['/bin/bash', '-c', 'echo error; exit 1']) logger.error.assert_called_with('error\n') @@ -18,4 +16,4 @@ def test_run_subprocess(mocker): logger = logging.getLogger('tljh') mocker.patch.object(logger, 'debug') utils.run_subprocess(['/bin/bash', '-c', 'echo success']) - logger.debug.assert_called_with('success\n') \ No newline at end of file + logger.debug.assert_called_with('success\n') diff --git a/tljh/apt.py b/tljh/apt.py index 4612958..d8efd94 100644 --- a/tljh/apt.py +++ b/tljh/apt.py @@ -25,7 +25,14 @@ def add_source(name, source_url, section): distro is determined from /etc/os-release """ # lsb_release is not installed in most docker images by default - distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}'], stderr=subprocess.STDOUT).decode().strip() + distro = ( + subprocess.check_output( + ['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}'], + stderr=subprocess.STDOUT, + ) + .decode() + .strip() + ) line = f'deb {source_url} {distro} {section}\n' with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f: # Write out deb line only if it already doesn't exist @@ -46,8 +53,4 @@ def install_packages(packages): env = os.environ.copy() # Stop apt from asking questions! env['DEBIAN_FRONTEND'] = 'noninteractive' - utils.run_subprocess([ - 'apt-get', - 'install', - '--yes' - ] + packages, env=env) + utils.run_subprocess(['apt-get', 'install', '--yes'] + packages, env=env) diff --git a/tljh/conda.py b/tljh/conda.py index 7ef4099..47cf831 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -30,10 +30,14 @@ def check_miniconda_version(prefix, version): Return true if a miniconda install with version exists at prefix """ try: - installed_version = subprocess.check_output([ - os.path.join(prefix, 'bin', 'conda'), - '-V' - ], stderr=subprocess.STDOUT).decode().strip().split()[1] + installed_version = ( + subprocess.check_output( + [os.path.join(prefix, 'bin', 'conda'), '-V'], stderr=subprocess.STDOUT + ) + .decode() + .strip() + .split()[1] + ) return V(installed_version) >= V(version) except (subprocess.CalledProcessError, FileNotFoundError): # Conda doesn't exist @@ -71,9 +75,7 @@ def fix_permissions(prefix): Run after each install command. """ - utils.run_subprocess( - ["chown", "-R", f"{os.getuid()}:{os.getgid()}", prefix] - ) + utils.run_subprocess(["chown", "-R", f"{os.getuid()}:{os.getgid()}", prefix]) utils.run_subprocess(["chmod", "-R", "o-w", prefix]) @@ -81,12 +83,7 @@ def install_miniconda(installer_path, prefix): """ Install miniconda with installer at installer_path under prefix """ - utils.run_subprocess([ - '/bin/bash', - installer_path, - '-u', '-b', - '-p', prefix - ]) + utils.run_subprocess(['/bin/bash', installer_path, '-u', '-b', '-p', prefix]) # fix permissions on initial install # a few files have the wrong ownership and permissions initially # when the installer is run as root @@ -106,21 +103,30 @@ def ensure_conda_packages(prefix, packages): # Explicitly do *not* capture stderr, since that's not always JSON! # Scripting conda is a PITA! # FIXME: raise different exception when using - raw_output = subprocess.check_output(conda_executable + [ - 'install', - '-c', 'conda-forge', # Make customizable if we ever need to - '--json', - '--prefix', abspath - ] + packages).decode() + raw_output = subprocess.check_output( + conda_executable + + [ + 'install', + '-c', + 'conda-forge', # Make customizable if we ever need to + '--json', + '--prefix', + abspath, + ] + + packages + ).decode() # `conda install` outputs JSON lines for fetch updates, # and a undelimited output at the end. There is no reasonable way to # parse this outside of this kludge. - filtered_output = '\n'.join([ - l for l in raw_output.split('\n') - # Sometimes the JSON messages start with a \x00. The lstrip removes these. - # conda messages seem to randomly throw \x00 in places for no reason - if not l.lstrip('\x00').startswith('{"fetch"') - ]) + filtered_output = '\n'.join( + [ + l + for l in raw_output.split('\n') + # Sometimes the JSON messages start with a \x00. The lstrip removes these. + # conda messages seem to randomly throw \x00 in places for no reason + if not l.lstrip('\x00').startswith('{"fetch"') + ] + ) output = json.loads(filtered_output.lstrip('\x00')) if 'success' in output and output['success'] == True: return diff --git a/tljh/config.py b/tljh/config.py index 0609c4f..01957d0 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -247,7 +247,9 @@ def check_hub_ready(): base_url = base_url[:-1] if base_url[-1] == '/' else base_url http_port = load_config()['http']['port'] try: - r = requests.get('http://127.0.0.1:%d%s/hub/api' % (http_port, base_url), verify=False) + r = requests.get( + 'http://127.0.0.1:%d%s/hub/api' % (http_port, base_url), verify=False + ) return r.status_code == 200 except: return False @@ -306,13 +308,17 @@ def _is_list(item): def main(argv=None): if os.geteuid() != 0: print("tljh-config needs root privileges to run", file=sys.stderr) - print("Try using sudo before the tljh-config command you wanted to run", file=sys.stderr) + print( + "Try using sudo before the tljh-config command you wanted to run", + file=sys.stderr, + ) sys.exit(1) if argv is None: argv = sys.argv[1:] from .log import init_logging + try: init_logging() except Exception as e: @@ -321,75 +327,48 @@ def main(argv=None): argparser = argparse.ArgumentParser() argparser.add_argument( - '--config-path', - default=CONFIG_FILE, - help='Path to TLJH config.yaml file' + '--config-path', default=CONFIG_FILE, help='Path to TLJH config.yaml file' ) subparsers = argparser.add_subparsers(dest='action') - show_parser = subparsers.add_parser( - 'show', - help='Show current configuration' - ) + show_parser = subparsers.add_parser('show', help='Show current configuration') - unset_parser = subparsers.add_parser( - 'unset', - help='Unset a configuration property' - ) + unset_parser = subparsers.add_parser('unset', help='Unset a configuration property') unset_parser.add_argument( - 'key_path', - help='Dot separated path to configuration key to unset' + 'key_path', help='Dot separated path to configuration key to unset' ) - set_parser = subparsers.add_parser( - 'set', - help='Set a configuration property' - ) + set_parser = subparsers.add_parser('set', help='Set a configuration property') set_parser.add_argument( - 'key_path', - help='Dot separated path to configuration key to set' - ) - set_parser.add_argument( - 'value', - help='Value to set the configuration key to' + 'key_path', help='Dot separated path to configuration key to set' ) + set_parser.add_argument('value', help='Value to set the configuration key to') add_item_parser = subparsers.add_parser( - 'add-item', - help='Add a value to a list for a configuration property' + 'add-item', help='Add a value to a list for a configuration property' ) add_item_parser.add_argument( - 'key_path', - help='Dot separated path to configuration key to add value to' - ) - add_item_parser.add_argument( - 'value', - help='Value to add to the configuration key' + 'key_path', help='Dot separated path to configuration key to add value to' ) + add_item_parser.add_argument('value', help='Value to add to the configuration key') remove_item_parser = subparsers.add_parser( - 'remove-item', - help='Remove a value from a list for a configuration property' + 'remove-item', help='Remove a value from a list for a configuration property' ) remove_item_parser.add_argument( - 'key_path', - help='Dot separated path to configuration key to remove value from' - ) - remove_item_parser.add_argument( - 'value', - help='Value to remove from key_path' + 'key_path', help='Dot separated path to configuration key to remove value from' ) + remove_item_parser.add_argument('value', help='Value to remove from key_path') reload_parser = subparsers.add_parser( - 'reload', - help='Reload a component to apply configuration change' + 'reload', help='Reload a component to apply configuration change' ) reload_parser.add_argument( 'component', choices=('hub', 'proxy'), help='Which component to reload', default='hub', - nargs='?' + nargs='?', ) args = argparser.parse_args(argv) @@ -397,16 +376,13 @@ def main(argv=None): if args.action == 'show': show_config(args.config_path) elif args.action == 'set': - set_config_value(args.config_path, args.key_path, - parse_value(args.value)) + set_config_value(args.config_path, args.key_path, parse_value(args.value)) elif args.action == 'unset': unset_config_value(args.config_path, args.key_path) elif args.action == 'add-item': - add_config_value(args.config_path, args.key_path, - parse_value(args.value)) + add_config_value(args.config_path, args.key_path, parse_value(args.value)) elif args.action == 'remove-item': - remove_config_value(args.config_path, args.key_path, - parse_value(args.value)) + remove_config_value(args.config_path, args.key_path, parse_value(args.value)) elif args.action == 'reload': reload_component(args.component) else: diff --git a/tljh/configurer.py b/tljh/configurer.py index 36bb69d..f22b01c 100644 --- a/tljh/configurer.py +++ b/tljh/configurer.py @@ -20,16 +20,9 @@ default = { 'base_url': '/', 'auth': { 'type': 'firstuseauthenticator.FirstUseAuthenticator', - 'FirstUseAuthenticator': { - 'create_users': False - } - }, - 'users': { - 'allowed': [], - 'banned': [], - 'admin': [], - 'extra_user_groups': {} + 'FirstUseAuthenticator': {'create_users': False}, }, + 'users': {'allowed': [], 'banned': [], 'admin': [], 'extra_user_groups': {}}, 'limits': { 'memory': None, 'cpu': None, @@ -65,12 +58,10 @@ default = { 'every': 60, 'concurrency': 5, 'users': False, - 'max_age': 0 + 'max_age': 0, }, - 'configurator': { - 'enabled': False - } - } + 'configurator': {'enabled': False}, + }, } @@ -189,7 +180,9 @@ def update_auth(c, config): if not (auth_key[0] == auth_key[0].upper() and isinstance(auth_value, dict)): if auth_key == 'type': continue - raise ValueError(f"Error: auth.{auth_key} was ignored, it didn't look like a valid configuration") + raise ValueError( + f"Error: auth.{auth_key} was ignored, it didn't look like a valid configuration" + ) class_name = auth_key class_config_to_set = auth_value class_config = c[class_name] @@ -255,9 +248,7 @@ def set_cull_idle_service(config): """ Set Idle Culler service """ - cull_cmd = [ - sys.executable, '-m', 'jupyterhub_idle_culler' - ] + cull_cmd = [sys.executable, '-m', 'jupyterhub_idle_culler'] cull_config = config['services']['cull'] print() @@ -283,8 +274,10 @@ def set_configurator(config): """ HERE = os.path.abspath(os.path.dirname(__file__)) configurator_cmd = [ - sys.executable, "-m", "jupyterhub_configurator.app", - f"--Configurator.config_file={HERE}/jupyterhub_configurator_config.py" + sys.executable, + "-m", + "jupyterhub_configurator.app", + f"--Configurator.config_file={HERE}/jupyterhub_configurator_config.py", ] configurator_service = { 'name': 'configurator', diff --git a/tljh/hooks.py b/tljh/hooks.py index b8c7808..8da7e11 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -22,6 +22,7 @@ def tljh_extra_user_pip_packages(): """ pass + @hookspec def tljh_extra_hub_pip_packages(): """ @@ -29,6 +30,7 @@ def tljh_extra_hub_pip_packages(): """ pass + @hookspec def tljh_extra_apt_packages(): """ @@ -38,6 +40,7 @@ def tljh_extra_apt_packages(): """ pass + @hookspec def tljh_custom_jupyterhub_config(c): """ @@ -48,6 +51,7 @@ def tljh_custom_jupyterhub_config(c): """ pass + @hookspec def tljh_config_post_install(config): """ @@ -60,6 +64,7 @@ def tljh_config_post_install(config): """ pass + @hookspec def tljh_post_install(): """ @@ -70,10 +75,11 @@ def tljh_post_install(): """ pass + @hookspec def tljh_new_user_create(username): """ Script to be executed after a new user has been added. This can be arbitrary Python code. """ - pass \ No newline at end of file + pass diff --git a/tljh/installer.py b/tljh/installer.py index a5177ef..e0ad0ec 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -73,11 +73,10 @@ def ensure_jupyterhub_service(prefix): with open(os.path.join(HERE, 'systemd-units', 'jupyterhub.service')) as f: hub_unit_template = f.read() - with open(os.path.join(HERE, 'systemd-units', 'traefik.service')) as f: traefik_unit_template = f.read() - #Set up proxy / hub secret token if it is not already setup + # Set up proxy / hub secret token if it is not already setup proxy_secret_path = os.path.join(STATE_DIR, 'traefik-api.secret') if not os.path.exists(proxy_secret_path): with open(proxy_secret_path, 'w') as f: @@ -103,7 +102,6 @@ def ensure_jupyterhub_service(prefix): systemd.enable_service('traefik') - def ensure_jupyterhub_package(prefix): """ Install JupyterHub into our conda environment if needed. @@ -117,11 +115,7 @@ def ensure_jupyterhub_package(prefix): # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # build-essential is also generally useful to everyone involved, and required for pycurl - apt.install_packages([ - 'libssl-dev', - 'libcurl4-openssl-dev', - 'build-essential' - ]) + apt.install_packages(['libssl-dev', 'libcurl4-openssl-dev', 'build-essential']) conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True) conda.ensure_pip_packages( @@ -135,7 +129,7 @@ def ensure_jupyterhub_package(prefix): "jupyterhub-tmpauthenticator==0.6.*", "oauthenticator==14.*", "jupyterhub-idle-culler==1.*", - "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba" + "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", ], upgrade=True, ) @@ -283,9 +277,9 @@ def ensure_jupyterhub_running(times=20): # Everything else should immediately abort raise except requests.ConnectionError: - # Hub isn't up yet, sleep & loop - time.sleep(1) - continue + # Hub isn't up yet, sleep & loop + time.sleep(1) + continue except Exception: # Everything else should immediately abort raise @@ -312,7 +306,9 @@ def ensure_symlinks(prefix): # tljh-config exists that isn't ours. We should *not* delete this file, # instead we throw an error and abort. Deleting files owned by other people # while running as root is dangerous, especially with symlinks involved. - raise FileExistsError(f'/usr/bin/tljh-config exists but is not a symlink to {tljh_config_src}') + raise FileExistsError( + f'/usr/bin/tljh-config exists but is not a symlink to {tljh_config_src}' + ) else: # We have a working symlink, so do nothing return @@ -343,17 +339,21 @@ def run_plugin_actions(plugin_manager): # Install apt packages apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages()))) if apt_packages: - logger.info('Installing {} apt packages collected from plugins: {}'.format( - len(apt_packages), ' '.join(apt_packages) - )) + logger.info( + 'Installing {} apt packages collected from plugins: {}'.format( + len(apt_packages), ' '.join(apt_packages) + ) + ) apt.install_packages(apt_packages) # Install hub pip packages hub_pip_packages = list(set(itertools.chain(*hook.tljh_extra_hub_pip_packages()))) if hub_pip_packages: - logger.info('Installing {} hub pip packages collected from plugins: {}'.format( - len(hub_pip_packages), ' '.join(hub_pip_packages) - )) + logger.info( + 'Installing {} hub pip packages collected from plugins: {}'.format( + len(hub_pip_packages), ' '.join(hub_pip_packages) + ) + ) conda.ensure_pip_packages( HUB_ENV_PREFIX, hub_pip_packages, @@ -363,17 +363,21 @@ def run_plugin_actions(plugin_manager): # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) if conda_packages: - logger.info('Installing {} user conda packages collected from plugins: {}'.format( - len(conda_packages), ' '.join(conda_packages) - )) + logger.info( + 'Installing {} user conda packages collected from plugins: {}'.format( + len(conda_packages), ' '.join(conda_packages) + ) + ) conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) # Install pip packages user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) if user_pip_packages: - logger.info('Installing {} user pip packages collected from plugins: {}'.format( - len(user_pip_packages), ' '.join(user_pip_packages) - )) + logger.info( + 'Installing {} user pip packages collected from plugins: {}'.format( + len(user_pip_packages), ' '.join(user_pip_packages) + ) + ) conda.ensure_pip_packages( USER_ENV_PREFIX, user_pip_packages, @@ -409,28 +413,22 @@ def ensure_config_yaml(plugin_manager): def main(): from .log import init_logging + init_logging() argparser = argparse.ArgumentParser() argparser.add_argument( - '--admin', - nargs='*', - action='append', - help='List of usernames set to be admin' + '--admin', nargs='*', action='append', help='List of usernames set to be admin' ) argparser.add_argument( '--user-requirements-txt-url', - help='URL to a requirements.txt file that should be installed in the user environment' - ) - argparser.add_argument( - '--plugin', - nargs='*', - help='Plugin pip-specs to install' + help='URL to a requirements.txt file that should be installed in the user environment', ) + argparser.add_argument('--plugin', nargs='*', help='Plugin pip-specs to install') argparser.add_argument( '--progress-page-server-pid', type=int, - help='The pid of the progress page server' + help='The pid of the progress page server', ) args = argparser.parse_args() diff --git a/tljh/normalize.py b/tljh/normalize.py index bba2b41..f51a4d4 100644 --- a/tljh/normalize.py +++ b/tljh/normalize.py @@ -10,7 +10,7 @@ def generate_system_username(username): If username < 26 char, we just return it. Else, we hash the username, truncate username at - 26 char, append a '-' and first add 5char of hash. + 26 char, append a '-' and first add 5char of hash. This makes sure our usernames are always under 32char. """ @@ -19,6 +19,5 @@ def generate_system_username(username): userhash = hashlib.sha256(username.encode('utf-8')).hexdigest() return '{username_trunc}-{hash}'.format( - username_trunc=username[:26], - hash=userhash[:5] - ) \ No newline at end of file + username_trunc=username[:26], hash=userhash[:5] + ) diff --git a/tljh/systemd.py b/tljh/systemd.py index fe85321..803c69c 100644 --- a/tljh/systemd.py +++ b/tljh/systemd.py @@ -13,10 +13,7 @@ def reload_daemon(): Makes systemd discover new units. """ - subprocess.run([ - 'systemctl', - 'daemon-reload' - ], check=True) + subprocess.run(['systemctl', 'daemon-reload'], check=True) def install_unit(name, unit, path='/etc/systemd/system'): @@ -31,43 +28,28 @@ def uninstall_unit(name, path='/etc/systemd/system'): """ Uninstall unit with given name """ - subprocess.run([ - 'rm', - os.path.join(path, name) - ], check=True) + subprocess.run(['rm', os.path.join(path, name)], check=True) def start_service(name): """ Start service with given name. """ - subprocess.run([ - 'systemctl', - 'start', - name - ], check=True) + subprocess.run(['systemctl', 'start', name], check=True) def stop_service(name): """ Start service with given name. """ - subprocess.run([ - 'systemctl', - 'stop', - name - ], check=True) + subprocess.run(['systemctl', 'stop', name], check=True) def restart_service(name): """ Restart service with given name. """ - subprocess.run([ - 'systemctl', - 'restart', - name - ], check=True) + subprocess.run(['systemctl', 'restart', name], check=True) def enable_service(name): @@ -76,11 +58,7 @@ def enable_service(name): This most likely makes the service start on bootup """ - subprocess.run([ - 'systemctl', - 'enable', - name - ], check=True) + subprocess.run(['systemctl', 'enable', name], check=True) def disable_service(name): @@ -89,11 +67,7 @@ def disable_service(name): This most likely makes the service start on bootup """ - subprocess.run([ - 'systemctl', - 'disable', - name - ], check=True) + subprocess.run(['systemctl', 'disable', name], check=True) def check_service_active(name): @@ -101,25 +75,18 @@ def check_service_active(name): Check if a service is currently active (running) """ try: - subprocess.run([ - 'systemctl', - 'is-active', - name - ], check=True) + subprocess.run(['systemctl', 'is-active', name], check=True) return True except subprocess.CalledProcessError: return False + def check_service_enabled(name): """ Check if a service is enabled """ try: - subprocess.run([ - 'systemctl', - 'is-enabled', - name - ], check=True) + subprocess.run(['systemctl', 'is-enabled', name], check=True) return True except subprocess.CalledProcessError: return False diff --git a/tljh/traefik.py b/tljh/traefik.py index aa4799f..6fe1533 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -29,6 +29,7 @@ checksums = { "linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1" } + def checksum_file(path): """Compute the sha256 checksum of a path""" hasher = hashlib.sha256() @@ -37,16 +38,13 @@ def checksum_file(path): hasher.update(chunk) return hasher.hexdigest() + def fatal_error(e): # Retry only when connection is reset or we think we didn't download entire file return str(e) != "ContentTooShort" and not isinstance(e, ConnectionResetError) -@backoff.on_exception( - backoff.expo, - Exception, - max_tries=2, - giveup=fatal_error -) + +@backoff.on_exception(backoff.expo, Exception, max_tries=2, giveup=fatal_error) def ensure_traefik_binary(prefix): """Download and install the traefik binary to a location identified by a prefix path such as '/opt/tljh/hub/'""" traefik_bin = os.path.join(prefix, "bin", "traefik") @@ -150,4 +148,3 @@ def ensure_traefik_config(state_dir): # ensure acme.json exists and is private with open(os.path.join(state_dir, "acme.json"), "a") as f: os.fchmod(f.fileno(), 0o600) - diff --git a/tljh/user.py b/tljh/user.py index edd2d90..2080b7c 100644 --- a/tljh/user.py +++ b/tljh/user.py @@ -25,17 +25,9 @@ def ensure_user(username): # User doesn't exist, time to create! pass - subprocess.check_call([ - 'useradd', - '--create-home', - username - ]) + subprocess.check_call(['useradd', '--create-home', username]) - subprocess.check_call([ - 'chmod', - 'o-rwx', - expanduser(f'~{username}') - ]) + subprocess.check_call(['chmod', 'o-rwx', expanduser(f'~{username}')]) pm = get_plugin_manager() pm.hook.tljh_new_user_create(username=username) @@ -51,22 +43,14 @@ def remove_user(username): # User doesn't exist, nothing to do return - subprocess.check_call([ - 'deluser', - '--quiet', - username - ]) + subprocess.check_call(['deluser', '--quiet', username]) def ensure_group(groupname): """ Ensure given group exists """ - subprocess.check_call([ - 'groupadd', - '--force', - groupname - ]) + subprocess.check_call(['groupadd', '--force', groupname]) def remove_group(groupname): @@ -79,11 +63,7 @@ def remove_group(groupname): # Group doesn't exist, nothing to do return - subprocess.check_call([ - 'delgroup', - '--quiet', - groupname - ]) + subprocess.check_call(['delgroup', '--quiet', groupname]) def ensure_user_group(username, groupname): @@ -96,12 +76,7 @@ def ensure_user_group(username, groupname): if username in group.gr_mem: return - subprocess.check_call([ - 'gpasswd', - '--add', - username, - groupname - ]) + subprocess.check_call(['gpasswd', '--add', username, groupname]) def remove_user_group(username, groupname): @@ -112,9 +87,4 @@ def remove_user_group(username, groupname): if username not in group.gr_mem: return - subprocess.check_call([ - 'gpasswd', - '--delete', - username, - groupname - ]) + subprocess.check_call(['gpasswd', '--delete', username, groupname]) diff --git a/tljh/user_creating_spawner.py b/tljh/user_creating_spawner.py index 6490596..8fe80ce 100644 --- a/tljh/user_creating_spawner.py +++ b/tljh/user_creating_spawner.py @@ -5,12 +5,14 @@ from systemdspawner import SystemdSpawner from traitlets import Dict, Unicode, List from jupyterhub_configurator.mixins import ConfiguratorSpawnerMixin + class CustomSpawner(SystemdSpawner): """ SystemdSpawner with user creation on spawn. FIXME: Remove this somehow? """ + user_groups = Dict(key_trait=Unicode(), value_trait=List(Unicode()), config=True) def start(self): @@ -34,12 +36,15 @@ class CustomSpawner(SystemdSpawner): user.ensure_user_group(system_username, group) return super().start() + cfg = configurer.load_config() # Use the jupyterhub-configurator mixin only if configurator is enabled # otherwise, any bugs in the configurator backend will stop new user spawns! if cfg['services']['configurator']['enabled']: # Dynamically create the Spawner class using `type`(https://docs.python.org/3/library/functions.html?#type), # based on whether or not it should inherit from ConfiguratorSpawnerMixin - UserCreatingSpawner = type('UserCreatingSpawner', (ConfiguratorSpawnerMixin, CustomSpawner), {}) + UserCreatingSpawner = type( + 'UserCreatingSpawner', (ConfiguratorSpawnerMixin, CustomSpawner), {} + ) else: UserCreatingSpawner = type('UserCreatingSpawner', (CustomSpawner,), {}) diff --git a/tljh/utils.py b/tljh/utils.py index 2b4d085..658c8ff 100644 --- a/tljh/utils.py +++ b/tljh/utils.py @@ -24,20 +24,26 @@ def run_subprocess(cmd, *args, **kwargs): and failed output directly to the user's screen """ logger = logging.getLogger('tljh') - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs) + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, *args, **kwargs + ) printable_command = ' '.join(cmd) if proc.returncode != 0: # Our process failed! Show output to the user - logger.error('Ran {command} with exit code {code}'.format( - command=printable_command, code=proc.returncode - )) + logger.error( + 'Ran {command} with exit code {code}'.format( + command=printable_command, code=proc.returncode + ) + ) logger.error(proc.stdout.decode()) raise subprocess.CalledProcessError(cmd=cmd, returncode=proc.returncode) else: # This goes into installer.log - logger.debug('Ran {command} with exit code {code}'.format( - command=printable_command, code=proc.returncode - )) + logger.debug( + 'Ran {command} with exit code {code}'.format( + command=printable_command, code=proc.returncode + ) + ) # This produces multi line log output, unfortunately. Not sure how to fix. # For now, prioritizing human readability over machine readability. logger.debug(proc.stdout.decode())