Merge pull request #876 from consideRatio/pr/add-upgrade-tests

maint: add upgrade test from main branch, latest release, and 0.2.0

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
       - run:
           name: Check upgrade testing
           command: |
-            if [ "$CIRCLE_BRANCH" == "master" ]; then
+            if [ "$CIRCLE_BRANCH" == "main" ]; then
                 echo "On master, no upgrade to test..."
                 circleci-agent step halt
             else

.github/integration-test.py

@@ -122,10 +122,16 @@ def copy_to_container(container_name, src_path, dest_path):
 
 
 def run_test(
-    image_name, test_name, bootstrap_pip_spec, test_files, upgrade, installer_args
+    image_name,
+    test_name,
+    bootstrap_pip_spec,
+    test_files,
+    upgrade_from,
+    installer_args,
 ):
     """
-    Wrapper that sets up tljh with installer_args & runs test_name
+    Starts a new container based on image_name, runs the bootstrap script to
+    setup tljh with installer_args, and runs test_name.
     """
     stop_container(test_name)
     run_systemd_image(image_name, test_name, bootstrap_pip_spec)
@@ -144,12 +150,26 @@ def run_test(
     print(container_check_output(["logs", test_name]).decode())
     print(f"--- End of logs from the container: {test_name}")
 
-    # Install TLJH from the default branch first to test upgrades
+    # To test upgrades, we run a bootstrap.py script two times instead of one,
-    if upgrade:
+    # where the initial run first installs some older version.
+    #
+    # We want to support testing a PR by upgrading from "main", "latest" (latest
+    # released version), and from a previous major-like version.
+    #
+    # FIXME: We currently always rely on the main branch's bootstrap.py script.
+    #        Realistically, we should run previous versions of the bootstrap
+    #        script which also installs previous versions of TLJH.
+    #
+    #        2023-04-15 Erik observed that https://tljh.jupyter.org/bootstrap.py
+    #        is referencing to the master (now main) branch which didn't seem
+    #        obvious, thinking it could have been the latest released version
+    #        also.
+    #
+    if upgrade_from:
         run_container_command(
-            test_name, "curl -L https://tljh.jupyter.org/bootstrap.py | python3 -"
+            test_name,
+            f"curl -L https://tljh.jupyter.org/bootstrap.py | python3 - --version={upgrade_from}",
         )
-
     run_container_command(test_name, f"python3 /srv/src/bootstrap.py {installer_args}")
 
     # Install pkgs from requirements in hub's pip, where
@@ -192,9 +212,11 @@ def main():
         dest="build_args",
     )
 
-    subparsers.add_parser("stop-container").add_argument("container_name")
+    stop_container_parser = subparsers.add_parser("stop-container")
+    stop_container_parser.add_argument("container_name")
 
-    subparsers.add_parser("start-container").add_argument("container_name")
+    start_container_parser = subparsers.add_parser("start-container")
+    start_container_parser.add_argument("container_name")
 
     run_parser = subparsers.add_parser("run")
     run_parser.add_argument("container_name")
@@ -207,7 +229,7 @@ def main():
 
     run_test_parser = subparsers.add_parser("run-test")
     run_test_parser.add_argument("--installer-args", default="")
-    run_test_parser.add_argument("--upgrade", action="store_true")
+    run_test_parser.add_argument("--upgrade-from", default="")
     run_test_parser.add_argument(
         "--bootstrap-pip-spec", nargs="?", default="", type=str
     )
@@ -227,7 +249,7 @@ def main():
             args.test_name,
             args.bootstrap_pip_spec,
             args.test_files,
-            args.upgrade,
+            args.upgrade_from,
             args.installer_args,
         )
     elif args.action == "show-logs":

.github/workflows/integration-test.yaml

@@ -8,14 +8,12 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "**.rst"
       - ".github/workflows/*"
       - "!.github/workflows/integration-test.yaml"
   push:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "**.rst"
       - ".github/workflows/*"
       - "!.github/workflows/integration-test.yaml"
     branches-ignore:
@@ -24,58 +22,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  # This job is used as a workaround to a limitation when using a matrix of
-  # variations that a job should be executed against. The limitation is that a
-  # matrix once defined can't include any conditions.
-  #
-  # What this job does before our real test job with a matrix of variations run,
-  # is to decide on that matrix of variations a conditional logic of our choice.
-  #
-  # For more details, see this excellent stack overflow answer:
-  # https://stackoverflow.com/a/65434401/2220152
-  #
-  decide-on-test-jobs-to-run:
-    name: Decide on test jobs to run
-    runs-on: ubuntu-latest
-
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-
-    steps:
-      # Currently, this logic filters out a matrix entry equaling a specific git
-      # reference identified by "dont_run_on_ref".
-      - name: Decide on test jobs to run
-        id: set-matrix
-        run: |
-          matrix_post_filter=$(
-            echo "$matrix_include_pre_filter" \
-              | yq e --output-format=json '.' - \
-              | jq -c '{"include": map( . | select(.dont_run_on_ref != "${{ github.ref }}" ))}'
-          )
-          echo "matrix=$matrix_post_filter" >> $GITHUB_OUTPUT
-
-          echo "The subsequent job's matrix are:"
-          echo $matrix_post_filter | jq -C '.'
-        env:
-          matrix_include_pre_filter: |
-            - name: "Int. tests: Debian 11, Py 3.9"
-              distro_image: "debian:11"
-              runs_on: "ubuntu-22.04"
-              extra_flags: ""
-            - name: "Int. tests: Ubuntu 20.04, Py 3.8"
-              distro_image: "ubuntu:20.04"
-              extra_flags: ""
-            - name: "Int. tests: Ubuntu 22.04 Py 3.10"
-              distro_image: "ubuntu:22.04"
-              extra_flags: ""
-            - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade"
-              distro_image: "ubuntu:22.04"
-              extra_flags: --upgrade
-              dont_run_on_ref: refs/heads/master
-
   integration-tests:
-    needs: decide-on-test-jobs-to-run
-
     # integration tests run in a container,
     # not in the worker, so this version is not relevant to the tests
     # and can be the same for all tested versions
@@ -84,7 +31,27 @@ jobs:
     name: ${{ matrix.name }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.decide-on-test-jobs-to-run.outputs.matrix) }}
+      matrix:
+        include:
+          - name: "Debian 11, Py 3.9"
+            distro_image: "debian:11"
+            runs_on: "ubuntu-22.04"
+            extra_flags: ""
+          - name: "Ubuntu 20.04, Py 3.8"
+            distro_image: "ubuntu:20.04"
+            extra_flags: ""
+          - name: "Ubuntu 22.04 Py 3.10"
+            distro_image: "ubuntu:22.04"
+            extra_flags: ""
+          - name: "Ubuntu 22.04, Py 3.10, from main"
+            distro_image: "ubuntu:22.04"
+            extra_flags: --upgrade-from=main
+          - name: "Ubuntu 22.04, Py 3.10, from latest"
+            distro_image: "ubuntu:22.04"
+            extra_flags: --upgrade-from=latest
+          - name: "Ubuntu 22.04, Py 3.10, from 0.2.0"
+            distro_image: "ubuntu:22.04"
+            extra_flags: --upgrade-from=0.2.0
 
     steps:
       - uses: actions/checkout@v3
@@ -107,7 +74,7 @@ jobs:
           # integration-tests/test_bootstrap.py will build and start containers
           # based on this environment variable. This is similar to how
           # .github/integration-test.py build-image can take a --build-arg
-          # setting the base image.
+          # setting the base image via a Dockerfile ARG.
           BASE_IMAGE: ${{ matrix.distro_image }}
 
       # We build a docker image from wherein we will work

.github/workflows/unit-test.yaml

@@ -8,14 +8,12 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "**.rst"
       - ".github/workflows/*"
       - "!.github/workflows/unit-test.yaml"
   push:
     paths-ignore:
       - "docs/**"
       - "**.md"
-      - "**.rst"
       - ".github/workflows/*"
       - "!.github/workflows/unit-test.yaml"
     branches-ignore:
@@ -42,10 +40,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: "Unit tests: Ubuntu 20.04, Py 3.9"
+          - name: "Ubuntu 20.04, Py 3.9"
             ubuntu_version: "20.04"
             python_version: "3.9"
-          - name: "Unit tests: Ubuntu 22.04, Py 3.10"
+          - name: "Ubuntu 22.04, Py 3.10"
             ubuntu_version: "22.04"
             python_version: "3.10"
 
@@ -97,5 +95,4 @@ jobs:
         run: pytest --verbose --maxfail=2 --color=yes --durations=10 --cov=tljh tests/
         timeout-minutes: 15
 
-      - name: Upload code coverage stats
+      - uses: codecov/codecov-action@v3
-        run: codecov

README.md

@@ -2,7 +2,7 @@
 
 [![Documentation build status](https://img.shields.io/readthedocs/the-littlest-jupyterhub?logo=read-the-docs)](https://tljh.jupyter.org/en/latest/?badge=latest)
 [![GitHub Workflow Status - Test](https://img.shields.io/github/workflow/status/jupyterhub/the-littlest-jupyterhub/Unit%20tests?logo=github&label=tests)](https://github.com/jupyterhub/the-littlest-jupyterhub/actions)
-[![Test coverage of code](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/master/graph/badge.svg)](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub)
+[![Test coverage of code](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub/branch/main/graph/badge.svg)](https://codecov.io/gh/jupyterhub/the-littlest-jupyterhub)
 [![GitHub](https://img.shields.io/badge/issue_tracking-github-blue?logo=github)](https://github.com/jupyterhub/the-littlest-jupyterhub/issues)
 [![Discourse](https://img.shields.io/badge/help_forum-discourse-blue?logo=discourse)](https://discourse.jupyter.org/c/jupyterhub/tljh)
 [![Gitter](https://img.shields.io/badge/social_chat-gitter-blue?logo=gitter)](https://gitter.im/jupyterhub/jupyterhub)

bootstrap/bootstrap.py

@@ -26,7 +26,7 @@ Environment variables:
                                 installing the tljh installer. Pass the values
                                 yes or no.
 
-Command line flags:
+Command line flags, from "bootstrap.py --help":
 
     The bootstrap.py script accept the following command line flags. All other
     flags are passed through to the tljh installer without interception by this
@@ -36,6 +36,11 @@ Command line flags:
                             logs can be accessed during installation. If this is
                             passed, it will pass --progress-page-server-pid=<pid>
                             to the tljh installer for later termination.
+    --version               TLJH version or Git reference. Default 'latest' is
+                            the most recent release. Partial versions can be
+                            specified, for example '1', '1.0' or '1.0.0'. You
+                            can also pass a branch name such as 'main' or a
+                            commit hash.
 """
 from argparse import ArgumentParser
 import os
@@ -340,8 +345,23 @@ def main():
     """
     distro, version = ensure_host_system_can_install_tljh()
 
-    parser = ArgumentParser()
+    parser = ArgumentParser(
-    parser.add_argument("--show-progress-page", action="store_true")
+        description=(
+            "The bootstrap.py script accept the following command line flags. "
+            "All other flags are passed through to the tljh installer without "
+            "interception by this script."
+        )
+    )
+    parser.add_argument(
+        "--show-progress-page",
+        action="store_true",
+        help=(
+            "Starts a local web server listening on port 80 where logs can be "
+            "accessed during installation. If this is passed, it will pass "
+            "--progress-page-server-pid=<pid> to the tljh installer for later "
+            "termination."
+        ),
+    )
     parser.add_argument(
         "--version",
         default="latest",
@@ -356,10 +376,10 @@ def main():
 
     # Various related constants
     install_prefix = os.environ.get("TLJH_INSTALL_PREFIX", "/opt/tljh")
-    hub_prefix = os.path.join(install_prefix, "hub")
+    hub_env_prefix = os.path.join(install_prefix, "hub")
-    python_bin = os.path.join(hub_prefix, "bin", "python3")
+    hub_env_python = os.path.join(hub_env_prefix, "bin", "python3")
-    pip_bin = os.path.join(hub_prefix, "bin", "pip")
+    hub_env_pip = os.path.join(hub_env_prefix, "bin", "pip")
-    initial_setup = not os.path.exists(python_bin)
+    initial_setup = not os.path.exists(hub_env_python)
 
     # Attempt to start a web server to serve a progress page reporting
     # installation progress.
@@ -451,18 +471,18 @@ def main():
             env=apt_get_adjusted_env,
         )
 
-        logger.info("Setting up virtual environment at {}".format(hub_prefix))
+        logger.info("Setting up virtual environment at {}".format(hub_env_prefix))
-        os.makedirs(hub_prefix, exist_ok=True)
+        os.makedirs(hub_env_prefix, exist_ok=True)
-        run_subprocess(["python3", "-m", "venv", hub_prefix])
+        run_subprocess(["python3", "-m", "venv", hub_env_prefix])
 
     # Upgrade pip
     # Keep pip version pinning in sync with the one in unit-test.yml!
     # See changelog at https://pip.pypa.io/en/latest/news/#changelog
     logger.info("Upgrading pip...")
-    run_subprocess([pip_bin, "install", "--upgrade", "pip==21.3.*"])
+    run_subprocess([hub_env_pip, "install", "--upgrade", "pip==21.3.*"])
 
     # Install/upgrade TLJH installer
-    tljh_install_cmd = [pip_bin, "install", "--upgrade"]
+    tljh_install_cmd = [hub_env_pip, "install", "--upgrade"]
     if os.environ.get("TLJH_BOOTSTRAP_DEV", "no") == "yes":
         logger.info("Selected TLJH_BOOTSTRAP_DEV=yes...")
         tljh_install_cmd.append("--editable")
@@ -484,7 +504,9 @@ def main():
 
     # Run TLJH installer
     logger.info("Running TLJH installer...")
-    os.execv(python_bin, [python_bin, "-m", "tljh.installer"] + tljh_installer_flags)
+    os.execv(
+        hub_env_python, [hub_env_python, "-m", "tljh.installer"] + tljh_installer_flags
+    )
 
 
 if __name__ == "__main__":

dev-requirements.txt

@@ -1,4 +1,3 @@
 pytest
 pytest-cov
 pytest-mock
-codecov