From ed334ac050fe3673a09696a96d288497628d2409 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 08:39:52 +0200 Subject: [PATCH 1/8] ci/docs: finalize transition to main branch --- .circleci/config.yml | 2 +- .github/workflows/integration-test.yaml | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 622622e..d344bc3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,7 +113,7 @@ jobs: - run: name: Check upgrade testing command: | - if [ "$CIRCLE_BRANCH" == "master" ]; then + if [ "$CIRCLE_BRANCH" == "main" ]; then echo "On master, no upgrade to test..." circleci-agent step halt else diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 940340d..b0ba46a 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -71,7 +71,7 @@ jobs: - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade" distro_image: "ubuntu:22.04" extra_flags: --upgrade - dont_run_on_ref: refs/heads/master + dont_run_on_ref: refs/heads/main integration-tests: needs: decide-on-test-jobs-to-run diff --git a/README.md b/README.md index 46f2d19..d16de87 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Documentation build status](https://img.shields.io/readthedocs/the-littlest-jupyterhub?logo=read-the-docs)](https://tljh.jupyter.org/en/latest/?badge=latest) [![GitHub Workflow Status - Test](https://img.shields.io/github/workflow/status/jupyterhub/the-littlest-jupyterhub/Unit%20tests?logo=github&label=tests)](https://github.com/jupyterhub/the-littlest-jupyterhub/actions) -[![Test coverage of code](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/master/graph/badge.svg)](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub) +[![Test coverage of code](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/main/graph/badge.svg)](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub) [![GitHub](https://img.shields.io/badge/issue_tracking-github-blue?logo=github)](https://github.com/jupyterhub/the-littlest-jupyterhub/issues) [![Discourse](https://img.shields.io/badge/help_forum-discourse-blue?logo=discourse)](https://discourse.jupyter.org/c/jupyterhub/tljh) [![Gitter](https://img.shields.io/badge/social_chat-gitter-blue?logo=gitter)](https://gitter.im/jupyterhub/jupyterhub) From 7859e7f01b1bd30d7a358ae4c316469df03b08dc Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 08:50:20 +0200 Subject: [PATCH 2/8] refactor: name pip_bin etc to clarify the python environment --- bootstrap/bootstrap.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 67cf46c..29f7d56 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -356,10 +356,10 @@ def main(): # Various related constants install_prefix = os.environ.get("TLJH_INSTALL_PREFIX", "/opt/tljh") - hub_prefix = os.path.join(install_prefix, "hub") - python_bin = os.path.join(hub_prefix, "bin", "python3") - pip_bin = os.path.join(hub_prefix, "bin", "pip") - initial_setup = not os.path.exists(python_bin) + hub_env_prefix = os.path.join(install_prefix, "hub") + hub_env_python = os.path.join(hub_env_prefix, "bin", "python3") + hub_env_pip = os.path.join(hub_env_prefix, "bin", "pip") + initial_setup = not os.path.exists(hub_env_python) # Attempt to start a web server to serve a progress page reporting # installation progress. @@ -451,18 +451,18 @@ def main(): env=apt_get_adjusted_env, ) - logger.info("Setting up virtual environment at {}".format(hub_prefix)) - os.makedirs(hub_prefix, exist_ok=True) - run_subprocess(["python3", "-m", "venv", hub_prefix]) + logger.info("Setting up virtual environment at {}".format(hub_env_prefix)) + os.makedirs(hub_env_prefix, exist_ok=True) + run_subprocess(["python3", "-m", "venv", hub_env_prefix]) # Upgrade pip # Keep pip version pinning in sync with the one in unit-test.yml! # See changelog at https://pip.pypa.io/en/latest/news/#changelog logger.info("Upgrading pip...") - run_subprocess([pip_bin, "install", "--upgrade", "pip==21.3.*"]) + run_subprocess([hub_env_pip, "install", "--upgrade", "pip==21.3.*"]) # Install/upgrade TLJH installer - tljh_install_cmd = [pip_bin, "install", "--upgrade"] + tljh_install_cmd = [hub_env_pip, "install", "--upgrade"] if os.environ.get("TLJH_BOOTSTRAP_DEV", "no") == "yes": logger.info("Selected TLJH_BOOTSTRAP_DEV=yes...") tljh_install_cmd.append("--editable") @@ -484,7 +484,9 @@ def main(): # Run TLJH installer logger.info("Running TLJH installer...") - os.execv(python_bin, [python_bin, "-m", "tljh.installer"] + tljh_installer_flags) + os.execv( + hub_env_python, [hub_env_python, "-m", "tljh.installer"] + tljh_installer_flags + ) if __name__ == "__main__": From 17bbd6116b9fe83618030a6560b1c4c35b3d12e2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 10:27:39 +0200 Subject: [PATCH 3/8] bootstrap.py: update documentation for consistency --- .github/workflows/integration-test.yaml | 1 - bootstrap/bootstrap.py | 26 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index b0ba46a..8be0a08 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -71,7 +71,6 @@ jobs: - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade" distro_image: "ubuntu:22.04" extra_flags: --upgrade - dont_run_on_ref: refs/heads/main integration-tests: needs: decide-on-test-jobs-to-run diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 29f7d56..ef9f602 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -26,7 +26,7 @@ Environment variables: installing the tljh installer. Pass the values yes or no. -Command line flags: +Command line flags, from "bootstrap.py --help": The bootstrap.py script accept the following command line flags. All other flags are passed through to the tljh installer without interception by this @@ -36,6 +36,11 @@ Command line flags: logs can be accessed during installation. If this is passed, it will pass --progress-page-server-pid= to the tljh installer for later termination. + --version TLJH version or Git reference. Default 'latest' is + the most recent release. Partial versions can be + specified, for example '1', '1.0' or '1.0.0'. You + can also pass a branch name such as 'main' or a + commit hash. """ from argparse import ArgumentParser import os @@ -340,8 +345,23 @@ def main(): """ distro, version = ensure_host_system_can_install_tljh() - parser = ArgumentParser() - parser.add_argument("--show-progress-page", action="store_true") + parser = ArgumentParser( + description=( + "The bootstrap.py script accept the following command line flags. " + "All other flags are passed through to the tljh installer without " + "interception by this script." + ) + ) + parser.add_argument( + "--show-progress-page", + action="store_true", + help=( + "Starts a local web server listening on port 80 where logs can be " + "accessed during installation. If this is passed, it will pass " + "--progress-page-server-pid= to the tljh installer for later " + "termination." + ), + ) parser.add_argument( "--version", default="latest", From 8e94bbe468407890dc2ee9f337bfbae38e41e550 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 10:28:38 +0200 Subject: [PATCH 4/8] ci: add upgrade tests from: main, latest, and 0.2.0 --- .github/integration-test.py | 42 +++++++++++++++++++------ .github/workflows/integration-test.yaml | 12 +++++-- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/.github/integration-test.py b/.github/integration-test.py index 25af580..f3c42b7 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -122,10 +122,16 @@ def copy_to_container(container_name, src_path, dest_path): def run_test( - image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args + image_name, + test_name, + bootstrap_pip_spec, + test_files, + upgrade_from, + installer_args, ): """ - Wrapper that sets up tljh with installer_args & runs test_name + Starts a new container based on image_name, runs the bootstrap script to + setup tljh with installer_args, and runs test_name. """ stop_container(test_name) run_systemd_image(image_name, test_name, bootstrap_pip_spec) @@ -144,12 +150,26 @@ def run_test( print(container_check_output(["logs", test_name]).decode()) print(f"--- End of logs from the container: {test_name}") - # Install TLJH from the default branch first to test upgrades - if upgrade: + # To test upgrades, we run a bootstrap.py script two times instead of one, + # where the initial run first installs some older version. + # + # We want to support testing a PR by upgrading from "main", "latest" (latest + # released version), and from a previous major-like version. + # + # FIXME: We currently always rely on the main branch's bootstrap.py script. + # Realistically, we should run previous versions of the bootstrap + # script which also installs previous versions of TLJH. + # + # 2023-04-15 Erik observed that https://tljh.jupyter.org/bootstrap.py + # is referencing to the master (now main) branch which didn't seem + # obvious, thinking it could have been the latest released version + # also. + # + if upgrade_from: run_container_command( - test_name, "curl -L https://tljh.jupyter.org/bootstrap.py | python3 -" + test_name, + f"curl -L https://tljh.jupyter.org/bootstrap.py | python3 - --version={upgrade_from}", ) - run_container_command(test_name, f"python3 /srv/src/bootstrap.py {installer_args}") # Install pkgs from requirements in hub's pip, where @@ -192,9 +212,11 @@ def main(): dest="build_args", ) - subparsers.add_parser("stop-container").add_argument("container_name") + stop_container_parser = subparsers.add_parser("stop-container") + stop_container_parser.add_argument("container_name") - subparsers.add_parser("start-container").add_argument("container_name") + start_container_parser = subparsers.add_parser("start-container") + start_container_parser.add_argument("container_name") run_parser = subparsers.add_parser("run") run_parser.add_argument("container_name") @@ -207,7 +229,7 @@ def main(): run_test_parser = subparsers.add_parser("run-test") run_test_parser.add_argument("--installer-args", default="") - run_test_parser.add_argument("--upgrade", action="store_true") + run_test_parser.add_argument("--upgrade-from", default="") run_test_parser.add_argument( "--bootstrap-pip-spec", nargs="?", default="", type=str ) @@ -227,7 +249,7 @@ def main(): args.test_name, args.bootstrap_pip_spec, args.test_files, - args.upgrade, + args.upgrade_from, args.installer_args, ) elif args.action == "show-logs": diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 8be0a08..cf8613c 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -68,9 +68,15 @@ jobs: - name: "Int. tests: Ubuntu 22.04 Py 3.10" distro_image: "ubuntu:22.04" extra_flags: "" - - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade" + - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=main" distro_image: "ubuntu:22.04" - extra_flags: --upgrade + extra_flags: --upgrade-from=main + - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=latest" + distro_image: "ubuntu:22.04" + extra_flags: --upgrade-from=latest + - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=0.2.0" + distro_image: "ubuntu:22.04" + extra_flags: --upgrade-from=0.2.0 integration-tests: needs: decide-on-test-jobs-to-run @@ -106,7 +112,7 @@ jobs: # integration-tests/test_bootstrap.py will build and start containers # based on this environment variable. This is similar to how # .github/integration-test.py build-image can take a --build-arg - # setting the base image. + # setting the base image via a Dockerfile ARG. BASE_IMAGE: ${{ matrix.distro_image }} # We build a docker image from wherein we will work From d531a127d841f726afd04c7e03c0728dea6014b4 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 10:44:14 +0200 Subject: [PATCH 5/8] ci: shorten job names --- .github/workflows/integration-test.yaml | 12 ++++++------ .github/workflows/unit-test.yaml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index cf8613c..eb318a4 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -58,23 +58,23 @@ jobs: echo $matrix_post_filter | jq -C '.' env: matrix_include_pre_filter: | - - name: "Int. tests: Debian 11, Py 3.9" + - name: "Debian 11, Py 3.9" distro_image: "debian:11" runs_on: "ubuntu-22.04" extra_flags: "" - - name: "Int. tests: Ubuntu 20.04, Py 3.8" + - name: "Ubuntu 20.04, Py 3.8" distro_image: "ubuntu:20.04" extra_flags: "" - - name: "Int. tests: Ubuntu 22.04 Py 3.10" + - name: "Ubuntu 22.04 Py 3.10" distro_image: "ubuntu:22.04" extra_flags: "" - - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=main" + - name: "Ubuntu 22.04, Py 3.10, from main" distro_image: "ubuntu:22.04" extra_flags: --upgrade-from=main - - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=latest" + - name: "Ubuntu 22.04, Py 3.10, from latest" distro_image: "ubuntu:22.04" extra_flags: --upgrade-from=latest - - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=0.2.0" + - name: "Ubuntu 22.04, Py 3.10, from 0.2.0" distro_image: "ubuntu:22.04" extra_flags: --upgrade-from=0.2.0 diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index a0acde8..f0ac849 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -42,10 +42,10 @@ jobs: fail-fast: false matrix: include: - - name: "Unit tests: Ubuntu 20.04, Py 3.9" + - name: "Ubuntu 20.04, Py 3.9" ubuntu_version: "20.04" python_version: "3.9" - - name: "Unit tests: Ubuntu 22.04, Py 3.10" + - name: "Ubuntu 22.04, Py 3.10" ubuntu_version: "22.04" python_version: "3.10" From 6ecbe083d537465547b60eb8b10851881bd84738 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 10:37:58 +0200 Subject: [PATCH 6/8] ci: use non-deprecated codecov action --- .github/workflows/unit-test.yaml | 3 +-- dev-requirements.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index f0ac849..f082164 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -97,5 +97,4 @@ jobs: run: pytest --verbose --maxfail=2 --color=yes --durations=10 --cov=tljh tests/ timeout-minutes: 15 - - name: Upload code coverage stats - run: codecov + - uses: codecov/codecov-action@v3 diff --git a/dev-requirements.txt b/dev-requirements.txt index 0c50f19..e1e3068 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,3 @@ pytest pytest-cov pytest-mock -codecov From f219acd041a9081ed604ebb4ef2e880c26fde954 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 10:56:44 +0200 Subject: [PATCH 7/8] ci: remove unimportant and complicated job filtering --- .github/workflows/integration-test.yaml | 78 +++++++------------------ 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index eb318a4..3cc8672 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -24,63 +24,7 @@ on: workflow_dispatch: jobs: - # This job is used as a workaround to a limitation when using a matrix of - # variations that a job should be executed against. The limitation is that a - # matrix once defined can't include any conditions. - # - # What this job does before our real test job with a matrix of variations run, - # is to decide on that matrix of variations a conditional logic of our choice. - # - # For more details, see this excellent stack overflow answer: - # https://stackoverflow.com/a/65434401/2220152 - # - decide-on-test-jobs-to-run: - name: Decide on test jobs to run - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - - steps: - # Currently, this logic filters out a matrix entry equaling a specific git - # reference identified by "dont_run_on_ref". - - name: Decide on test jobs to run - id: set-matrix - run: | - matrix_post_filter=$( - echo "$matrix_include_pre_filter" \ - | yq e --output-format=json '.' - \ - | jq -c '{"include": map( . | select(.dont_run_on_ref != "${{ github.ref }}" ))}' - ) - echo "matrix=$matrix_post_filter" >> $GITHUB_OUTPUT - - echo "The subsequent job's matrix are:" - echo $matrix_post_filter | jq -C '.' - env: - matrix_include_pre_filter: | - - name: "Debian 11, Py 3.9" - distro_image: "debian:11" - runs_on: "ubuntu-22.04" - extra_flags: "" - - name: "Ubuntu 20.04, Py 3.8" - distro_image: "ubuntu:20.04" - extra_flags: "" - - name: "Ubuntu 22.04 Py 3.10" - distro_image: "ubuntu:22.04" - extra_flags: "" - - name: "Ubuntu 22.04, Py 3.10, from main" - distro_image: "ubuntu:22.04" - extra_flags: --upgrade-from=main - - name: "Ubuntu 22.04, Py 3.10, from latest" - distro_image: "ubuntu:22.04" - extra_flags: --upgrade-from=latest - - name: "Ubuntu 22.04, Py 3.10, from 0.2.0" - distro_image: "ubuntu:22.04" - extra_flags: --upgrade-from=0.2.0 - integration-tests: - needs: decide-on-test-jobs-to-run - # integration tests run in a container, # not in the worker, so this version is not relevant to the tests # and can be the same for all tested versions @@ -89,7 +33,27 @@ jobs: name: ${{ matrix.name }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.decide-on-test-jobs-to-run.outputs.matrix) }} + matrix: + include: + - name: "Debian 11, Py 3.9" + distro_image: "debian:11" + runs_on: "ubuntu-22.04" + extra_flags: "" + - name: "Ubuntu 20.04, Py 3.8" + distro_image: "ubuntu:20.04" + extra_flags: "" + - name: "Ubuntu 22.04 Py 3.10" + distro_image: "ubuntu:22.04" + extra_flags: "" + - name: "Ubuntu 22.04, Py 3.10, from main" + distro_image: "ubuntu:22.04" + extra_flags: --upgrade-from=main + - name: "Ubuntu 22.04, Py 3.10, from latest" + distro_image: "ubuntu:22.04" + extra_flags: --upgrade-from=latest + - name: "Ubuntu 22.04, Py 3.10, from 0.2.0" + distro_image: "ubuntu:22.04" + extra_flags: --upgrade-from=0.2.0 steps: - uses: actions/checkout@v3 From acd420765a000574471f50eb2b6cf47bb35b460f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 15 Apr 2023 11:45:44 +0200 Subject: [PATCH 8/8] ci: cleanup no longer needed .rst file type reference --- .github/workflows/integration-test.yaml | 2 -- .github/workflows/unit-test.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 3cc8672..4c04c11 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -8,14 +8,12 @@ on: paths-ignore: - "docs/**" - "**.md" - - "**.rst" - ".github/workflows/*" - "!.github/workflows/integration-test.yaml" push: paths-ignore: - "docs/**" - "**.md" - - "**.rst" - ".github/workflows/*" - "!.github/workflows/integration-test.yaml" branches-ignore: diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index f082164..e93c0ce 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -8,14 +8,12 @@ on: paths-ignore: - "docs/**" - "**.md" - - "**.rst" - ".github/workflows/*" - "!.github/workflows/unit-test.yaml" push: paths-ignore: - "docs/**" - "**.md" - - "**.rst" - ".github/workflows/*" - "!.github/workflows/unit-test.yaml" branches-ignore: