ci: towards testing non-LTS ubuntu versions

.github/workflows/integration-test.yaml

@@ -60,22 +60,31 @@ jobs:
         env:
           matrix_include_pre_filter: |
             - name: "Int. tests: Ubuntu 18.04, Py 3.6"
-              ubuntu_version: 18.04
+              ubuntu_version: "18.04"
-              python_version: 3.6
+              python_version: "3.6"
               extra_flags: ""
             - name: "Int. tests: Ubuntu 20.04, Py 3.9"
-              ubuntu_version: 20.04
+              ubuntu_version: "20.04"
-              python_version: 3.9
+              python_version: "3.9"
               extra_flags: ""
             - name: "Int. tests: Ubuntu 20.04, Py 3.9, --upgrade"
-              ubuntu_version: 20.04
+              ubuntu_version: "20.04"
-              python_version: 3.9
+              python_version: "3.9"
               extra_flags: --upgrade
               dont_run_on_ref: refs/heads/master
 
   integration-tests:
     needs: decide-on-test-jobs-to-run
-    runs-on: ubuntu-${{ matrix.ubuntu_version }}
+
+    # runs-on can only be configured to the LTS releases of ubuntu (18.04,
+    # 20.04, ...), so if we want to test against the latest non-LTS release, we
+    # must compromise when configuring runs-on and configure runs-on to be the
+    # latest LTS release instead.
+    #
+    # This can have consequences because actions like actions/setup-python will
+    # mount cached installations associated with the chosen runs-on environment.
+    #
+    runs-on: ${{ format('ubuntu-{0}', matrix.runs_on || matrix.ubuntu_version) }}
 
     name: ${{ matrix.name }}
     strategy:
@@ -95,36 +104,39 @@ jobs:
 
       - name: Build systemd image
         run: |
-          .github/integration-test.py build-image --build-arg ubuntu_version=${{ matrix.ubuntu_version }}
+          .github/integration-test.py build-image \
+              --build-arg "ubuntu_version=${{ matrix.ubuntu_version }}"
 
       - name: Install pytest
-        run: |
+        run: python3 -m pip install pytest
-          python3 -m pip install pytest
 
       - name: Run bootstrap tests
         run: |
-          pytest --verbose --maxfail=2 --color=yes --capture=no integration-tests/test_bootstrap.py
+          pytest --verbose --maxfail=2 --color=yes --capture=no \
+              integration-tests/test_bootstrap.py
 
       - name: Run basic tests
         run: |
-          .github/integration-test.py run-test \
+          .github/integration-test.py run-test basic-tests \
-            --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
+              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
-            basic-tests test_hub.py test_proxy.py \
+              ${{ matrix.extra_flags }} \
-            test_install.py test_extensions.py \
+              test_hub.py \
-            ${{ matrix.extra_flags }}
+              test_proxy.py \
+              test_install.py \
+              test_extensions.py
 
       - name: Run admin tests
         run: |
-          .github/integration-test.py run-test \
+          .github/integration-test.py run-test admin-tests \
-            --installer-args "--admin admin:admin" \
+              --installer-args "--admin admin:admin" \
-            --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
+              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
-            basic-tests test_admin_installer.py \
+              ${{ matrix.extra_flags }} \
-            ${{ matrix.extra_flags }}
+              test_admin_installer.py
 
       - name: Run plugin tests
         run: |
-          .github/integration-test.py run-test \
+          .github/integration-test.py run-test plugin-tests \
-            --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
+              --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \
-            --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \
+              --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \
-            plugins test_simplest_plugin.py \
+              ${{ matrix.extra_flags }} \
-            ${{ matrix.extra_flags }}
+              test_simplest_plugin.py

.github/workflows/unit-test.yaml

@@ -26,7 +26,16 @@ on:
 jobs:
   unit-tests:
     name: ${{ matrix.name }}
-    runs-on: ubuntu-${{ matrix.ubuntu_version }}
+
+    # runs-on can only be configured to the LTS releases of ubuntu (18.04,
+    # 20.04, ...), so if we want to test against the latest non-LTS release, we
+    # must compromise when configuring runs-on and configure runs-on to be the
+    # latest LTS release instead.
+    #
+    # This can have consequences because actions like actions/setup-python will
+    # mount cached installations associated with the chosen runs-on environment.
+    #
+    runs-on: ${{ format('ubuntu-{0}', matrix.runs_on || matrix.ubuntu_version) }}
     container: ubuntu:${{ matrix.ubuntu_version }}
 
     strategy:
@@ -34,11 +43,11 @@ jobs:
       matrix:
         include:
           - name: "Unit tests: Ubuntu 18.04, Py 3.6"
-            ubuntu_version: 18.04
+            ubuntu_version: "18.04"
-            python_version: 3.6
+            python_version: "3.6"
           - name: "Unit tests: Ubuntu 20.04, Py 3.9"
-            ubuntu_version: 20.04
+            ubuntu_version: "20.04"
-            python_version: 3.9
+            python_version: "3.9"
 
     steps:
       - uses: actions/checkout@v2
@@ -47,16 +56,31 @@ jobs:
           python-version: ${{ matrix.python_version }}
 
       - name: Install venv, git and setup venv
+        timeout-minutes: 2
         run: |
-          apt-get update --yes && apt-get install --yes python3-venv git
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update
+          apt-get install --yes \
+              python3-venv \
+              git
+
           python3 -m venv /srv/venv
           echo '/srv/venv/bin' >> $GITHUB_PATH
 
-      - name: Cache pip deps
+      # WARNING: This action loads a cache of pip dependencies based on the
+      #          declared key, and it will save a cache for that key on job
+      #          completion. Make sure to update the key to bust the cache
+      #          properly if you make a change that should influence it.
+      - name: Load cached Python dependencies
         uses: actions/cache@v2
         with:
           path: /srv/venv/
-          key: pip-${{ matrix.ubuntu_version }}-${{ matrix.python_version }}-${{ hashFiles('*setup.py', '*dev-requirements.txt') }}
+          key: >-
+            pip-
+            ${{ matrix.runs_on }}-
+            ${{ matrix.ubuntu_version }}-
+            ${{ matrix.python_version }}-
+            ${{ hashFiles('setup.py', 'dev-requirements.txt', '.github/workflows/unit-test.yaml') }}
 
       - name: Install Python dependencies
         run: |
@@ -66,9 +90,7 @@ jobs:
           pip freeze
 
       - name: Run unit tests
-        run: |
+        run: pytest --verbose --maxfail=2 --color=yes --cov=tljh tests/
-          pytest --verbose --maxfail=2 --color=yes --cov=tljh tests/
 
       - name: Upload code coverage stats
-        run: |
+        run: codecov
-          codecov

integration-tests/Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:${ubuntu_version}
 
 # DEBIAN_FRONTEND is set to avoid being asked for input and hang during build:
 # https://anonoz.github.io/tech/2020/04/24/docker-build-stuck-tzdata.html
-RUN DEBIAN_FRONTEND=noninteractive \
+RUN export DEBIAN_FRONTEND=noninteractive \
  && apt-get update \
  && apt-get install --yes \
         systemd \