the-littlest-jupyterhub/.github/workflows/integration-test.yaml

# This is a GitHub workflow defining a set of jobs with a set of steps.
# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
#
name: Integration tests

on:
  pull_request:
    paths-ignore:
      - "docs/**"
      - "**.md"
      - "**.rst"
      - ".github/workflows/*"
      - "!.github/workflows/integration-test.yaml"
  push:
    paths-ignore:
      - "docs/**"
      - "**.md"
      - "**.rst"
      - ".github/workflows/*"
      - "!.github/workflows/integration-test.yaml"
    branches-ignore:
      - "dependabot/**"
      - "pre-commit-ci-update-config"
  workflow_dispatch:

jobs:
  # This job is used as a workaround to a limitation when using a matrix of
  # variations that a job should be executed against. The limitation is that a
  # matrix once defined can't include any conditions.
  #
  # What this job does before our real test job with a matrix of variations run,
  # is to decide on that matrix of variations a conditional logic of our choice.
  #
  # For more details, see this excellent stack overflow answer:
  # https://stackoverflow.com/a/65434401/2220152
  #
  decide-on-test-jobs-to-run:
    name: Decide on test jobs to run
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      # Currently, this logic filters out a matrix entry equaling a specific git
      # reference identified by "dont_run_on_ref".
      - name: Decide on test jobs to run
        id: set-matrix
        run: |
          matrix_post_filter=$(
            echo "$matrix_include_pre_filter" \
              | yq e --output-format=json '.' - \
              | jq -c '{"include": map( . | select(.dont_run_on_ref != "${{ github.ref }}" ))}'
          )
          echo "matrix=$matrix_post_filter" >> $GITHUB_OUTPUT

          echo "The subsequent job's matrix are:"
          echo $matrix_post_filter | jq -C '.'
        env:
          matrix_include_pre_filter: |
            - name: "Int. tests: Debian 11, Py 3.9"
              distro_image: "debian:11"
              runs_on: "ubuntu-22.04"
              extra_flags: ""
            - name: "Int. tests: Ubuntu 20.04, Py 3.8"
              distro_image: "ubuntu:20.04"
              extra_flags: ""
            - name: "Int. tests: Ubuntu 22.04 Py 3.10"
              distro_image: "ubuntu:22.04"
              extra_flags: ""
            - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=main"
              distro_image: "ubuntu:22.04"
              extra_flags: --upgrade-from=main
            - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=latest"
              distro_image: "ubuntu:22.04"
              extra_flags: --upgrade-from=latest
            - name: "Int. tests: Ubuntu 22.04, Py 3.10, --upgrade-from=0.2.0"
              distro_image: "ubuntu:22.04"
              extra_flags: --upgrade-from=0.2.0

  integration-tests:
    needs: decide-on-test-jobs-to-run

    # integration tests run in a container,
    # not in the worker, so this version is not relevant to the tests
    # and can be the same for all tested versions
    runs-on: ubuntu-22.04

    name: ${{ matrix.name }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.decide-on-test-jobs-to-run.outputs.matrix) }}

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Install pytest
        run: python3 -m pip install pytest

      # We abort pytest after two failures as a compromise between wanting to
      # avoid a flood of logs while still understanding if multiple tests would
      # fail.
      - name: Run bootstrap tests (Runs in/Builds ${{ matrix.distro_image }} derived image)
        run: |
          pytest --verbose --maxfail=2 --color=yes --durations=10 --capture=no \
              integration-tests/test_bootstrap.py
        timeout-minutes: 15
        env:
          # integration-tests/test_bootstrap.py will build and start containers
          # based on this environment variable. This is similar to how
          # .github/integration-test.py build-image can take a --build-arg
          # setting the base image via a Dockerfile ARG.
          BASE_IMAGE: ${{ matrix.distro_image }}

      # We build a docker image from wherein we will work
      - name: Build systemd image (Builds ${{ matrix.distro_image }} derived image)
        run: |
          .github/integration-test.py build-image \
              --build-arg "BASE_IMAGE=${{ matrix.distro_image }}"

      # FIXME: Make the logic below easier to follow.
      # - In short, setting BOOTSTRAP_PIP_SPEC here, specifies from what
      #   location the tljh python package should be installed from. In this
      #   GitHub Workflow's test job, we provide a remote reference to itself as
      #   found on GitHub - this could be the HEAD of a PR branch or the default
      #   branch on merge.
      #
      # Overview of how this logic influences the end result.
      # - integration-test.yaml:
      #   Runs integration-test.py by passing --bootstrap-pip-spec flag with a
      #   reference to the pull request on GitHub.
      # - integration-test.py:
      #   Starts a pre-build systemd container, setting the
      #   TLJH_BOOTSTRAP_PIP_SPEC based on its passed --bootstrap-pip-spec value.
      # - systemd container:
      #   Runs bootstrap.py
      # - bootstrap.py
      #   Makes use of TLJH_BOOTSTRAP_PIP_SPEC environment variable to install
      #   the tljh package from a given location, which could be a local git
      #   clone of this repo where setup.py resides, or a reference to some
      #   GitHub branch for example.
      - name: Set BOOTSTRAP_PIP_SPEC value
        run: |
          BOOTSTRAP_PIP_SPEC="git+https://github.com/$GITHUB_REPOSITORY.git@$GITHUB_REF"
          echo "BOOTSTRAP_PIP_SPEC=$BOOTSTRAP_PIP_SPEC" >> $GITHUB_ENV
          echo $BOOTSTRAP_PIP_SPEC

      - name: Run basic tests (Runs in ${{ matrix.distro_image }} derived image)
        run: |
          .github/integration-test.py run-test basic-tests \
              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
              ${{ matrix.extra_flags }} \
              test_hub.py \
              test_proxy.py \
              test_install.py \
              test_extensions.py
        timeout-minutes: 15

      - name: Run admin tests (Runs in ${{ matrix.distro_image }} derived image)
        run: |
          .github/integration-test.py run-test admin-tests \
              --installer-args "--admin admin:admin" \
              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
              ${{ matrix.extra_flags }} \
              test_admin_installer.py
        timeout-minutes: 15

      - name: Run plugin tests (Runs in ${{ matrix.distro_image }} derived image)
        run: |
          .github/integration-test.py run-test plugin-tests \
              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
              --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \
              ${{ matrix.extra_flags }} \
              test_simplest_plugin.py
        timeout-minutes: 15