From f25d35d7a0ce188e3628118fc5820f984ed2e0ed Mon Sep 17 00:00:00 2001 From: Connor Dibble Date: Wed, 3 Nov 2021 22:44:49 +0100 Subject: [PATCH 1/3] docs: notes for arm64 support --- docs/index.rst | 3 +-- docs/topic/installer-actions.rst | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4eab5cf..074a4d2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,8 +17,7 @@ might still make breaking changes that have no clear upgrade pathway. Installation ============ -The Littlest JupyterHub (TLJH) can run on any server that is running at least -**Ubuntu 18.04**. Earlier versions of Ubuntu are not supported. +The Littlest JupyterHub (TLJH) can run on any server that is running **Ubuntu 18.04** or **Ubuntu 20.04** on a amd64 or arm64 CPU architecture. Earlier versions of Ubuntu are not supported. We have a bunch of tutorials to get you started. - Tutorials to create a new server from scratch on a cloud provider & run TLJH diff --git a/docs/topic/installer-actions.rst b/docs/topic/installer-actions.rst index caad82a..56e0302 100644 --- a/docs/topic/installer-actions.rst +++ b/docs/topic/installer-actions.rst @@ -30,7 +30,7 @@ of `traefik `_. This virtual environment is completely manag User environment ================ -By default, a ``miniconda`` environment is installed in ``/opt/tljh/user``. This contains +By default, a ``mambaforge`` conda environment is installed in ``/opt/tljh/user``. This contains the notebook interface used to launch all users, and the various packages available to all users. The environment is owned by the ``root`` user. JupyterHub admins may use to ``sudo -E conda install`` or ``sudo -E pip install`` packages into this environment. From a91571dd222991a13ceeac59a8f7a4c75e7a92e4 Mon Sep 17 00:00:00 2001 From: Connor Dibble Date: Wed, 3 Nov 2021 22:47:57 +0100 Subject: [PATCH 2/3] Support Arm64 CPU architectures --- tests/test_conda.py | 10 +++++++--- tljh/installer.py | 11 +++++++++-- tljh/traefik.py | 19 ++++++++++++++----- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index a4db013..7632673 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -11,11 +11,15 @@ import tempfile @pytest.fixture(scope='module') def prefix(): """ - Provide a temporary directory with a conda environment + Provide a temporary directory with a mambaforge conda environment """ + # see https://github.com/conda-forge/miniforge/releases mambaforge_version = '4.10.3-7' - installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" - installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_version) + if os.uname().machine == 'aarch64': + installer_sha256 = "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2" + elif os.uname().machine == 'x86_64': + installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(v=mambaforge_version, arch=os.uname().machine) with tempfile.TemporaryDirectory() as tmpdir: with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: conda.install_miniconda(installer_path, tmpdir) diff --git a/tljh/installer.py b/tljh/installer.py index 3bb4e84..a5177ef 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -170,7 +170,14 @@ def ensure_user_environment(user_requirements_txt_file): # Install mambaforge using an installer from # https://github.com/conda-forge/miniforge/releases mambaforge_new_version = '4.10.3-7' - installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" + # Check system architecture, set appropriate installer checksum + if os.uname().machine == 'aarch64': + installer_sha256 = "ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2" + elif os.uname().machine == 'x86_64': + installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" + # Check OS, set appropriate string for conda installer path + if os.uname().sysname != 'Linux': + raise OSError("TLJH is only supported on Linux platforms.") # Then run `mamba --version` to get the conda and mamba versions # Keep these in sync with tests/test_conda.py::prefix mambaforge_conda_new_version = '4.10.3' @@ -185,7 +192,7 @@ def ensure_user_environment(user_requirements_txt_file): # If no prior miniconda installation is found, we can install a newer version else: logger.info('Downloading & setting up user environment...') - installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_new_version) + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(v=mambaforge_new_version, arch=os.uname().machine) with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda_version = '4.10.3' diff --git a/tljh/traefik.py b/tljh/traefik.py index 4c3151f..aa4799f 100644 --- a/tljh/traefik.py +++ b/tljh/traefik.py @@ -12,13 +12,21 @@ import toml from .config import CONFIG_DIR from tljh.configurer import load_config, _merge_dictionaries -# FIXME: support more than one platform here -plat = "linux-amd64" -traefik_version = "1.7.18" +# traefik 2.7.x is not supported yet, use v1.7.x for now +# see: https://github.com/jupyterhub/traefik-proxy/issues/97 +machine = os.uname().machine +if machine == 'aarch64': + plat = "linux-arm64" +elif machine == 'x86_64': + plat = "linux-amd64" +else: + raise OSError(f"Error. Platform: {os.uname().sysname} / {machine} Not supported.") +traefik_version = "1.7.33" # record sha256 hashes for supported platforms here checksums = { - "linux-amd64": "3c2d153d80890b6fc8875af9f8ced32c4d684e1eb5a46d9815337cb343dfd92e" + "linux-amd64": "314ffeaa4cd8ed6ab7b779e9b6773987819f79b23c28d7ab60ace4d3683c5935", + "linux-arm64": "0640fa665125efa6b598fc08c100178e24de66c5c6035ce5d75668d3dc3706e1" } def checksum_file(path): @@ -40,7 +48,7 @@ def fatal_error(e): giveup=fatal_error ) def ensure_traefik_binary(prefix): - """Download and install the traefik binary""" + """Download and install the traefik binary to a location identified by a prefix path such as '/opt/tljh/hub/'""" traefik_bin = os.path.join(prefix, "bin", "traefik") if os.path.exists(traefik_bin): checksum = checksum_file(traefik_bin) @@ -57,6 +65,7 @@ def ensure_traefik_binary(prefix): "https://github.com/containous/traefik/releases" f"/download/v{traefik_version}/traefik_{plat}" ) + print(f"Downloading traefik {traefik_version}...") # download the file response = requests.get(traefik_url) From 912f395538c9694f11ed65646b43865b0bada26b Mon Sep 17 00:00:00 2001 From: Connor Dibble Date: Wed, 3 Nov 2021 22:48:16 +0100 Subject: [PATCH 3/3] Add CircleCI test against Arm64 architecture --- .circleci/config.yml | 142 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..fdb7696 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,142 @@ +# We use circleci to test our support for the Arm64 architecture. This file +# should mimic what is done in the GitHub workflows. +# +# Configuration reference: https://circleci.com/docs/2.0/configuration-reference/ +# +version: 2.1 + +commands: + build_systemd_image: + steps: + - run: + name: build systemd image + command: | + .circleci/integration-test.py build-image + + basic_tests: + parameters: + # Whether or not we should run update tests + upgrade: + type: string + default: "" + + steps: + - run: + name: Run basic tests + command: | + if [ $CIRCLE_PR_USERNAME ]; then + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PR_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + else + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PROJECT_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + fi + + .circleci/integration-test.py run-test \ + --bootstrap-pip-spec "$BOOTSTRAP_PIP_SPEC" \ + basic-tests test_hub.py test_proxy.py \ + test_install.py test_extensions.py \ + << parameters.upgrade >> + + admin_tests: + parameters: + upgrade: + type: string + default: "" + + steps: + - run: + name: Run admin tests + command: | + if [ $CIRCLE_PR_USERNAME ]; then + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PR_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + else + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PROJECT_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + fi + + .circleci/integration-test.py run-test \ + --installer-args "--admin admin:admin" \ + --bootstrap-pip-spec $BOOTSTRAP_PIP_SPEC \ + basic-tests test_admin_installer.py \ + << parameters.upgrade >> + + plugin_tests: + parameters: + upgrade: + type: string + default: "" + + steps: + - run: + name: Run plugin tests + command: | + if [ $CIRCLE_PR_USERNAME ]; then + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PR_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + else + BOOTSTRAP_PIP_SPEC=git+https://github.com/$CIRCLE_PROJECT_USERNAME/the-littlest-jupyterhub.git@$CIRCLE_SHA1 + fi + + .circleci/integration-test.py run-test \ + --bootstrap-pip-spec $BOOTSTRAP_PIP_SPEC \ + --installer-args "--plugin /srv/src/integration-tests/plugins/simplest" \ + plugins test_simplest_plugin.py \ + << parameters.upgrade >> + + bootstrap_checks: + steps: + - run: + name: Run bootstrap checks + command: | + py.test integration-tests/test_bootstrap.py -s + + +jobs: + integration-test: + docker: + - image: docker:18.05.0-ce-git + + steps: + - run: + name: setup python3 + command: | + apk add --no-cache python3 pytest + - checkout + - setup_remote_docker + - build_systemd_image + - bootstrap_checks + - basic_tests + - admin_tests + - plugin_tests + + upgrade-test: + docker: + - image: docker:18.05.0-ce-git + + steps: + - run: + name: Check upgrade testing + command: | + if [ "$CIRCLE_BRANCH" == "master" ]; then + echo "On master, no upgrade to test..." + circleci-agent step halt + else + echo "PR detected, testing upgrade..." + fi + - run: + name: setup python3 + command: | + apk add --no-cache python3 pytest + - checkout + - setup_remote_docker + - build_systemd_image + - basic_tests: + upgrade: "--upgrade" + - admin_tests: + upgrade: "--upgrade" + - plugin_tests: + upgrade: "--upgrade" + +workflows: + version: 2 + all-tests: + jobs: + - integration-test + - upgrade-test