Make it easier to run multiple independent integration tests

Provide higher level run-test command in integration-tests.
This runs a number of pytest files in the same container
with the same installation.

.circleci/config.yml

@@ -68,38 +68,11 @@ jobs:
                     python3 .circleci/integration-test.py build-image
 
             - run:
-                name: start systemd image
+                name: Run hub tests
                 command: |
-                    python3 .circleci/integration-test.py start-container
+                    python3 .circleci/integration-test.py run-test test_hub.py
-
-            - run:
-                name: run tljh installer
-                command: |
-                    python3 .circleci/integration-test.py copy . /srv/src
-                    python3 .circleci/integration-test.py run 'python3 /srv/src/bootstrap/bootstrap.py'
 
 
-            - run:
-                name: switch to dummyauthenticator
-                command: |
-                    python3 .circleci/integration-test.py run '/opt/tljh/hub/bin/tljh-config set auth.type dummyauthenticator.DummyAuthenticator'
-                    python3 .circleci/integration-test.py run '/opt/tljh/hub/bin/tljh-config reload'
-
-            - run:
-                name: print systemd status + logs
-                command: |
-                    python3 .circleci/integration-test.py run 'journalctl --no-pager'
-                    python3 .circleci/integration-test.py run 'systemctl --no-pager status jupyterhub configurable-http-proxy'
-
-            - run:
-                name: install integration test requirements
-                command: |
-                    python3 .circleci/integration-test.py run 'python3 -m pip install -r /srv/src/integration-tests/requirements.txt'
-
-            - run:
-                name: run integration tests
-                command: |
-                    python3 .circleci/integration-test.py run 'python3 -m pytest -v /srv/src/integration-tests'
 
 workflows:
     version: 2

.circleci/integration-test.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import argparse
 import subprocess
 import os
@@ -32,14 +33,14 @@ def run_systemd_image(image_name, container_name):
     ])
 
 
-def remove_systemd_container(container_name):
+def stop_container(container_name):
     """
     Stop & remove docker container if it exists.
     """
     try:
         subprocess.check_output([
             'docker', 'inspect', container_name
-        ])
+        ], stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError:
         # No such container exists, nothing to do
         return
@@ -52,12 +53,17 @@ def run_container_command(container_name, cmd):
     """
     Run cmd in a running container with a bash shell
     """
-    subprocess.check_call([
+    proc = subprocess.run([
         'docker', 'exec',
         '-it', container_name,
         '/bin/bash', '-c', cmd
     ])
 
+    if proc.returncode != 0:
+        # Don't throw if command fails. This lets us continue next parts
+        # of tests. Not entirely sure this is the right thing to do though!
+        print(f'command {cmd} exited with return code {proc.returncode}')
+
 
 def copy_to_container(container_name, src_path, dest_path):
     """
@@ -69,13 +75,54 @@ def copy_to_container(container_name, src_path, dest_path):
     ])
 
 
+def run_test(image_name, test_name, test_files, installer_args):
+    """
+    Wrapper that sets up tljh with installer_args & runs test_name
+    """
+    stop_container(test_name)
+    run_systemd_image(image_name, test_name)
+
+    source_path = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir)
+    )
+
+    copy_to_container(test_name, source_path, '/srv/src')
+    run_container_command(
+        test_name,
+        f'python3 /srv/src/bootstrap/bootstrap.py {installer_args}'
+    )
+    run_container_command(
+        test_name,
+        'python3 -m pip install -r /srv/src/integration-tests/requirements.txt'
+    )
+    run_container_command(
+        test_name,
+        'python3 -m pytest -v {}'.format(
+            ' '.join([os.path.join('/srv/src/integration-tests/', f) for f in test_files])
+        )
+    )
+
+
+def show_logs(container_name):
+    """
+    Print logs from inside container to stdout
+    """
+    run_container_command(
+        container_name,
+        'journalctl --no-pager'
+    )
+    run_container_command(
+        container_name,
+        'systemctl --no-pager status jupyterhub configurable-http-proxy'
+    )
+
 def main():
     argparser = argparse.ArgumentParser()
     subparsers = argparser.add_subparsers(dest='action')
 
-    subparsers.add_parser('build-image')
+    subparsers.add_parser('stop-container').add_argument(
-    subparsers.add_parser('start-container')
+        'container_name'
-    subparsers.add_parser('stop-container')
+    )
     subparsers.add_parser('run').add_argument(
         'command',
     )
@@ -83,24 +130,28 @@ def main():
     copy_parser.add_argument('src')
     copy_parser.add_argument('dest')
 
+    run_test_parser = subparsers.add_parser('run-test')
+    run_test_parser.add_argument('--installer-args', default='')
+    run_test_parser.add_argument('test_name')
+    run_test_parser.add_argument('test_files', nargs='+')
+
+    show_logs_parser = subparsers.add_parser('show-logs')
+    show_logs_parser.add_argument('container_name')
+
     args = argparser.parse_args()
 
     image_name = 'tljh-systemd'
-    container_name = 'tljh-ci-run'
-    source_path = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), os.pardir, 'integration-tests')
-    )
 
-    if args.action == 'build-image':
+    if args.action == 'run-test':
-        build_systemd_image(image_name, source_path)
+        run_test(image_name, args.test_name, args.test_files, args.installer_args)
-    elif args.action == 'start-container':
+    elif args.action == 'show-logs':
-        run_systemd_image(image_name, container_name)
+        show_logs(args.container_name)
-    elif args.action == 'stop-container':
-        remove_systemd_container(container_name)
     elif args.action == 'run':
-        run_container_command(container_name, args.command)
+        run_container_command(args.container_name, args.command)
     elif args.action == 'copy':
-        copy_to_container(container_name, args.src, args.dest)
+        copy_to_container(args.container_name, args.src, args.dest)
+    elif args.action == 'stop-container':
+        stop_container(args.container_name)
 
 
 if __name__ == '__main__':

integration-tests/test_hub.py

@@ -29,6 +29,13 @@ async def test_user_code_execute():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
+
+    # FIXME: wait for reload to finish & hub to come up
+    # Should be part of tljh-config reload
+    await asyncio.sleep(1)
+
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -49,7 +56,7 @@ async def test_user_admin_add():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
@@ -79,6 +86,7 @@ async def test_user_admin_remove():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()