From 0018beca4f8656db53c01097171c9f5785f9f546 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 31 Jul 2021 20:44:23 +0100 Subject: [PATCH 1/7] Switch to mambaforge --- tests/test_conda.py | 10 +++++----- tljh/conda.py | 2 +- tljh/installer.py | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index 12dbd81..4e6adb0 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -13,14 +13,14 @@ def prefix(): """ Provide a temporary directory with a conda environment """ - miniconda_version = '4.7.10' - miniconda_installer_sha256 = "8a324adcc9eaf1c09e22a992bb6234d91a94146840ee6b11c114ecadafc68121" - installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(miniconda_version) + mambaforge_version = '4.10.3-3' + installer_sha256 = "a012c24e1cc3bcbe74a1e5693e510830e7c2956e85877b08d1e28707a0bd8d75" + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_version) with tempfile.TemporaryDirectory() as tmpdir: - with conda.download_miniconda_installer(installer_url, miniconda_installer_sha256) as installer_path: + with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: conda.install_miniconda(installer_path, tmpdir) conda.ensure_conda_packages(tmpdir, [ - 'conda==4.8.1' + 'conda==4.10.3' ]) yield tmpdir diff --git a/tljh/conda.py b/tljh/conda.py index 3fa8a87..9c8c749 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -94,7 +94,7 @@ def ensure_conda_packages(prefix, packages): """ Ensure packages (from conda-forge) are installed in the conda prefix. """ - conda_executable = [os.path.join(prefix, 'bin', 'python'), '-m', 'conda'] + conda_executable = [os.path.join(prefix, 'bin', 'mamba')] abspath = os.path.abspath(prefix) # Let subprocess errors propagate # Explicitly do *not* capture stderr, since that's not always JSON! diff --git a/tljh/installer.py b/tljh/installer.py index 2a68a19..2965292 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -171,21 +171,21 @@ def ensure_user_environment(user_requirements_txt_file): logger.info("Setting up user environment...") miniconda_old_version = '4.5.4' - miniconda_new_version = '4.7.10' - miniconda_installer_sha256 = "8a324adcc9eaf1c09e22a992bb6234d91a94146840ee6b11c114ecadafc68121" + mambaforge_new_version = '4.10.3-3' + installer_sha256 = "a012c24e1cc3bcbe74a1e5693e510830e7c2956e85877b08d1e28707a0bd8d75" - if conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version): - conda_version = '4.8.1' + if conda.check_miniconda_version(USER_ENV_PREFIX, mambaforge_new_version): + conda_version = '4.10.3' elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version): conda_version = '4.5.8' # If no prior miniconda installation is found, we can install a newer version else: logger.info('Downloading & setting up user environment...') # FIXME: allow using miniforge - installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(miniconda_new_version) - with conda.download_miniconda_installer(installer_url, miniconda_installer_sha256) as installer_path: + installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_new_version) + with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) - conda_version = '4.8.1' + conda_version = '4.10.3' conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. From d869540a6c48ec527eeb523cb41332cb6fd26612 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 31 Jul 2021 22:19:21 +0100 Subject: [PATCH 2/7] ensure mamba installed, add old miniconda check back --- tljh/installer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index 2965292..ff0c6b8 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -171,11 +171,16 @@ def ensure_user_environment(user_requirements_txt_file): logger.info("Setting up user environment...") miniconda_old_version = '4.5.4' + miniconda_new_version = '4.7.10' + mambaconda_new_version = '4.10.3' mambaforge_new_version = '4.10.3-3' installer_sha256 = "a012c24e1cc3bcbe74a1e5693e510830e7c2956e85877b08d1e28707a0bd8d75" + mamba_version = '0.15.2' - if conda.check_miniconda_version(USER_ENV_PREFIX, mambaforge_new_version): + if conda.check_miniconda_version(USER_ENV_PREFIX, mambaconda_new_version): conda_version = '4.10.3' + elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version): + conda_version = '4.8.1' elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version): conda_version = '4.5.8' # If no prior miniconda installation is found, we can install a newer version @@ -189,7 +194,8 @@ def ensure_user_environment(user_requirements_txt_file): conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. - 'conda==' + conda_version + 'conda==' + conda_version, + 'mamba==' + mamba_version, ]) conda.ensure_pip_requirements( From 1859a4690f5b0567e9d6f1323378994bfe10e826 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 31 Jul 2021 22:31:52 +0100 Subject: [PATCH 3/7] Lower test memory to 900m --- .github/integration-test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration-test.py b/.github/integration-test.py index dfbf65a..13bd0a2 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -31,7 +31,7 @@ def run_systemd_image(image_name, container_name, bootstrap_pip_spec): '--name', container_name, # This is the minimum VM size we support. # If we change this, need to change all other references to this number. - '--memory', '1G', + '--memory', '900m', ] if bootstrap_pip_spec: From e3bb87b99c3f9e9fc47be8833df7aec2f325e5ea Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 18 Oct 2021 20:31:41 +0100 Subject: [PATCH 4/7] Update Amazon docs: `t2.micro` works but recommend `t3.small` --- .github/integration-test.py | 4 ++-- docs/install/amazon.rst | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/integration-test.py b/.github/integration-test.py index 13bd0a2..7450628 100755 --- a/.github/integration-test.py +++ b/.github/integration-test.py @@ -29,8 +29,8 @@ def run_systemd_image(image_name, container_name, bootstrap_pip_spec): '--mount', 'type=bind,source=/sys/fs/cgroup,target=/sys/fs/cgroup', '--detach', '--name', container_name, - # This is the minimum VM size we support. - # If we change this, need to change all other references to this number. + # A bit less than 1GB to ensure TLJH runs on 1GB VMs. + # If this is changed all docs references to the required memory must be changed too. '--memory', '900m', ] diff --git a/docs/install/amazon.rst b/docs/install/amazon.rst index 4ee9c86..05adce5 100644 --- a/docs/install/amazon.rst +++ b/docs/install/amazon.rst @@ -76,8 +76,9 @@ Let's create the server on which we can run JupyterHub. `Next: Configure Instance Details` in the lower right corner. Check out our guide on How To :ref:`howto/admin/resource-estimation` to help pick - how much Memory / CPU your server needs. You need to have at least **1GB** of - RAM. The smallest instance that can fit this is a **t3.small**. + how much Memory / CPU your server needs. + We recommend you use a server with at leat 2GB of RAM, such as a **t3.small**. + However, if you need to minimise costs you can use a server with **1GB** RAM such as a **t2.micro**, but performance will be limited. You may wish to consult the listing `here `_ because it shows cost per hour. The **On Demand** price is the pertinent cost. From b3c07ba265dfe21fe8c08ee246ec6ed8f95786d3 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 18 Oct 2021 22:09:42 +0100 Subject: [PATCH 5/7] miniforge FIXME was fixed! Co-authored-by: Erik Sundell --- tljh/installer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tljh/installer.py b/tljh/installer.py index ff0c6b8..050bfd2 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -186,7 +186,6 @@ def ensure_user_environment(user_requirements_txt_file): # If no prior miniconda installation is found, we can install a newer version else: logger.info('Downloading & setting up user environment...') - # FIXME: allow using miniforge installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_new_version) with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) From 80b347349ad8e912b16a75369a72f2be0d22fb78 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 18 Oct 2021 22:33:06 +0100 Subject: [PATCH 6/7] Update to mambaforge_version 4.10.3-7 --- tests/test_conda.py | 4 ++-- tljh/installer.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index 4e6adb0..539e646 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -13,8 +13,8 @@ def prefix(): """ Provide a temporary directory with a conda environment """ - mambaforge_version = '4.10.3-3' - installer_sha256 = "a012c24e1cc3bcbe74a1e5693e510830e7c2956e85877b08d1e28707a0bd8d75" + mambaforge_version = '4.10.3-7' + installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-x86_64.sh".format(v=mambaforge_version) with tempfile.TemporaryDirectory() as tmpdir: with conda.download_miniconda_installer(installer_url, installer_sha256) as installer_path: diff --git a/tljh/installer.py b/tljh/installer.py index 050bfd2..05b4ce4 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -172,12 +172,16 @@ def ensure_user_environment(user_requirements_txt_file): miniconda_old_version = '4.5.4' miniconda_new_version = '4.7.10' - mambaconda_new_version = '4.10.3' - mambaforge_new_version = '4.10.3-3' - installer_sha256 = "a012c24e1cc3bcbe74a1e5693e510830e7c2956e85877b08d1e28707a0bd8d75" - mamba_version = '0.15.2' + # Install mambaforge using an installer from + # https://github.com/conda-forge/miniforge/releases + mambaforge_new_version = '4.10.3-7' + installer_sha256 = "fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474" + # Then run `mamba --version` to get the conda and mamba versions + # Keep these in sync with tests/test_conda.py::prefix + mambaforge_conda_new_version = '4.10.3' + mambaforge_mamba_version = '0.16.0' - if conda.check_miniconda_version(USER_ENV_PREFIX, mambaconda_new_version): + if conda.check_miniconda_version(USER_ENV_PREFIX, mambaforge_conda_new_version): conda_version = '4.10.3' elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version): conda_version = '4.8.1' @@ -194,7 +198,7 @@ def ensure_user_environment(user_requirements_txt_file): conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==' + conda_version, - 'mamba==' + mamba_version, + 'mamba==' + mambaforge_mamba_version, ]) conda.ensure_pip_requirements( From f61b7fb7a9d8a3563676048e07ba2deefd9a14a8 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 18 Oct 2021 22:59:48 +0100 Subject: [PATCH 7/7] Fix typo --- docs/install/amazon.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install/amazon.rst b/docs/install/amazon.rst index 05adce5..e8408dc 100644 --- a/docs/install/amazon.rst +++ b/docs/install/amazon.rst @@ -77,7 +77,7 @@ Let's create the server on which we can run JupyterHub. Check out our guide on How To :ref:`howto/admin/resource-estimation` to help pick how much Memory / CPU your server needs. - We recommend you use a server with at leat 2GB of RAM, such as a **t3.small**. + We recommend you use a server with at least 2GB of RAM, such as a **t3.small**. However, if you need to minimise costs you can use a server with **1GB** RAM such as a **t2.micro**, but performance will be limited. You may wish to consult the listing `here `_