mirror of
https://github.com/jupyterhub/the-littlest-jupyterhub.git
synced 2025-12-18 21:54:05 +08:00
Avoid downgrading user-env conda/mamba
- Only check lower bound on conda/mamba, upgrade unbounded if not matched (let conda apply upper bound according to existing pins, such as Python) - handle missing mamba - avoid upgrading Python by aborting the install, instead of keeping old envs - minimum supported Python for user env is 3.9 - Fix output reporting of conda install step (no need for json capture when we don't parse the output - exit codes will do)
This commit is contained in:
Min RK
@@ -13,7 +13,6 @@ import time
|
||||
import requests
|
||||
|
||||
from tljh import utils
|
||||
from tljh.utils import parse_version as V
|
||||
|
||||
|
||||
def sha256_file(fname):
|
||||
@@ -29,47 +28,20 @@ def sha256_file(fname):
|
||||
return hash_sha256.hexdigest()
|
||||
|
||||
|
||||
def check_miniconda_version(prefix, version):
|
||||
"""
|
||||
Return true if a miniconda install with version exists at prefix
|
||||
"""
|
||||
versions = get_mamba_versions(prefix)
|
||||
if "conda" not in versions:
|
||||
return False
|
||||
|
||||
return V(versions["conda"]) >= V(version)
|
||||
|
||||
|
||||
def get_mamba_versions(prefix):
|
||||
"""Parse `mamba --version` output into a dict
|
||||
|
||||
which looks like:
|
||||
|
||||
mamba 1.1.0
|
||||
conda 22.11.1
|
||||
|
||||
into:
|
||||
|
||||
{
|
||||
"mamba": "1.1.0",
|
||||
"conda": "22.11.1",
|
||||
}
|
||||
"""
|
||||
def get_conda_package_versions(prefix):
|
||||
"""Get conda package versions, via `conda list --json`"""
|
||||
versions = {}
|
||||
try:
|
||||
out = (
|
||||
subprocess.check_output(
|
||||
[os.path.join(prefix, "bin", "mamba"), "--version"],
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
.decode()
|
||||
.strip()
|
||||
out = subprocess.check_output(
|
||||
[os.path.join(prefix, "bin", "conda"), "list", "--json"],
|
||||
text=True,
|
||||
)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return versions
|
||||
for line in out.strip().splitlines():
|
||||
pkg, version = line.split()
|
||||
versions[pkg] = version
|
||||
|
||||
packages = json.loads(out)
|
||||
for package in packages:
|
||||
versions[package["name"]] = package["version"]
|
||||
return versions
|
||||
|
||||
|
||||
@@ -133,39 +105,24 @@ def ensure_conda_packages(prefix, packages):
|
||||
Note that conda seem to update dependencies by default, so there is probably
|
||||
no need to have a update parameter exposed for this function.
|
||||
"""
|
||||
conda_executable = [os.path.join(prefix, "bin", "mamba")]
|
||||
conda_executable = os.path.join(prefix, "bin", "mamba")
|
||||
if not os.path.isfile(conda_executable):
|
||||
# fallback on conda if mamba is not present (e.g. for mamba to install itself)
|
||||
conda_executable = os.path.join(prefix, "bin", "conda")
|
||||
|
||||
abspath = os.path.abspath(prefix)
|
||||
# Let subprocess errors propagate
|
||||
# Explicitly do *not* capture stderr, since that's not always JSON!
|
||||
# Scripting conda is a PITA!
|
||||
# FIXME: raise different exception when using
|
||||
raw_output = subprocess.check_output(
|
||||
conda_executable
|
||||
+ [
|
||||
|
||||
utils.run_subprocess(
|
||||
[
|
||||
conda_executable,
|
||||
"install",
|
||||
"-c",
|
||||
"conda-forge", # Make customizable if we ever need to
|
||||
"--json",
|
||||
"--prefix",
|
||||
abspath,
|
||||
]
|
||||
+ packages
|
||||
).decode()
|
||||
# `conda install` outputs JSON lines for fetch updates,
|
||||
# and a undelimited output at the end. There is no reasonable way to
|
||||
# parse this outside of this kludge.
|
||||
filtered_output = "\n".join(
|
||||
[
|
||||
l
|
||||
for l in raw_output.split("\n")
|
||||
# Sometimes the JSON messages start with a \x00. The lstrip removes these.
|
||||
# conda messages seem to randomly throw \x00 in places for no reason
|
||||
if not l.lstrip("\x00").startswith('{"fetch"')
|
||||
]
|
||||
+ packages,
|
||||
)
|
||||
output = json.loads(filtered_output.lstrip("\x00"))
|
||||
if "success" in output and output["success"] == True:
|
||||
return
|
||||
fix_permissions(prefix)
|
||||
|
||||
|
||||
|
||||
@@ -164,11 +164,15 @@ MAMBAFORGE_CHECKSUMS = {
|
||||
"aarch64": "96191001f27e0cc76612d4498d34f9f656d8a7dddee44617159e42558651479c",
|
||||
"x86_64": "16c7d256de783ceeb39970e675efa4a8eb830dcbb83187f1197abfea0bf07d30",
|
||||
}
|
||||
# run `mamba --version` to get the conda and mamba versions
|
||||
# conda/mamba will be _upgraded_ to these versions, if they differ from what's in
|
||||
# the mambaforge distribution
|
||||
MAMBAFORGE_MAMBA_VERSION = "1.1.0"
|
||||
MAMBAFORGE_CONDA_VERSION = "22.11.1"
|
||||
|
||||
# minimum versions of packages
|
||||
MINIMUM_VERSIONS = {
|
||||
# if conda/mamba are lower than this, upgrade them before installing the user packages
|
||||
"mamba": "0.16.0",
|
||||
"conda": "4.10",
|
||||
# minimum Python version (if not matched, abort to avoid big disruptive updates)
|
||||
"python": "3.9",
|
||||
}
|
||||
|
||||
|
||||
def _mambaforge_url(version=MAMBAFORGE_VERSION, arch=None):
|
||||
@@ -196,54 +200,71 @@ def ensure_user_environment(user_requirements_txt_file):
|
||||
Set up user conda environment with required packages
|
||||
"""
|
||||
logger.info("Setting up user environment...")
|
||||
# note: these must be in descending order
|
||||
conda_upgrade_versions = {
|
||||
# format: "conda version": (conda_version, mamba_version),
|
||||
# mambaforge 4.10.3-7 (2023-03-21)
|
||||
"22.11.1": (MAMBAFORGE_CONDA_VERSION, MAMBAFORGE_MAMBA_VERSION),
|
||||
# tljh up to 0.2.0 (since 2021-10-18)
|
||||
"4.10.3": ("4.10.3", "0.16.0"),
|
||||
}
|
||||
|
||||
# Check OS, set appropriate string for conda installer path
|
||||
if os.uname().sysname != "Linux":
|
||||
raise OSError("TLJH is only supported on Linux platforms.")
|
||||
found_conda = False
|
||||
have_versions = conda.get_mamba_versions(USER_ENV_PREFIX)
|
||||
have_conda_version = have_versions.get("conda")
|
||||
if have_conda_version:
|
||||
logger.info(
|
||||
f"Found prefix at {USER_ENV_PREFIX}, with conda/mamba({have_versions})"
|
||||
)
|
||||
for check_version, conda_mamba_version in conda_upgrade_versions.items():
|
||||
if V(have_conda_version) >= V(check_version):
|
||||
found_conda = True
|
||||
conda_version, mamba_version = conda_mamba_version
|
||||
break
|
||||
|
||||
if not found_conda:
|
||||
if os.path.exists(USER_ENV_PREFIX):
|
||||
logger.warning(
|
||||
f"Found prefix at {USER_ENV_PREFIX}, but too old or missing conda/mamba ({have_versions}). Upgrading from mambaforge."
|
||||
)
|
||||
# FIXME: should this fail? I'm not sure how destructive it is
|
||||
# Check the existing environment for what to do
|
||||
package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
|
||||
|
||||
# Case 1: no existing environment
|
||||
if not package_versions:
|
||||
# 1a. no environment, but prefix exists.
|
||||
# Abort to avoid clobbering something we don't recognize
|
||||
if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX):
|
||||
msg = f"Found non-empty directory that is not a conda install in {USER_ENV_PREFIX}. Please remove it (or rename it to preserve files) and run tljh again."
|
||||
logger.error(msg)
|
||||
raise OSError(msg)
|
||||
|
||||
# 1b. No environment, directory empty or doesn't exist
|
||||
# start fresh install
|
||||
logger.info("Downloading & setting up user environment...")
|
||||
installer_url, installer_sha256 = _mambaforge_url()
|
||||
with conda.download_miniconda_installer(
|
||||
installer_url, installer_sha256
|
||||
) as installer_path:
|
||||
conda.install_miniconda(installer_path, USER_ENV_PREFIX)
|
||||
conda_version = MAMBAFORGE_CONDA_VERSION
|
||||
mamba_version = MAMBAFORGE_MAMBA_VERSION
|
||||
package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
|
||||
# quick sanity check: we should have conda and mamba!
|
||||
assert "conda" in package_versions
|
||||
assert "mamba" in package_versions
|
||||
|
||||
conda.ensure_conda_packages(
|
||||
USER_ENV_PREFIX,
|
||||
[
|
||||
# Conda's latest version is on conda much more so than on PyPI.
|
||||
"conda==" + conda_version,
|
||||
"mamba==" + mamba_version,
|
||||
],
|
||||
)
|
||||
# next, check Python
|
||||
python_version = package_versions["python"]
|
||||
logger.debug(f"Found python={python_version} in {USER_ENV_PREFIX}")
|
||||
if V(python_version) < V(MINIMUM_VERSIONS["python"]):
|
||||
msg = (
|
||||
f"TLJH requires Python >={MINIMUM_VERSIONS['python']}, found python={python_version} in {USER_ENV_PREFIX}."
|
||||
f"\nPlease upgrade Python (may be highly disruptive!), or remove/rename {USER_ENV_PREFIX} to allow TLJH to make a fresh install."
|
||||
f"\nYou can use `{USER_ENV_PREFIX}/bin/conda list` to save your current list of packages."
|
||||
)
|
||||
logger.error(msg)
|
||||
raise ValueError(msg)
|
||||
|
||||
# at this point, we know we have an env ready with conda and are going to start installing
|
||||
# first, check if we should upgrade/install conda and/or mamba
|
||||
to_upgrade = []
|
||||
for pkg in ("conda", "mamba"):
|
||||
version = package_versions.get(pkg)
|
||||
min_version = MINIMUM_VERSIONS[pkg]
|
||||
if not version:
|
||||
logger.warning(f"{USER_ENV_PREFIX} is missing {pkg}, installing it...")
|
||||
to_upgrade.append(pkg)
|
||||
else:
|
||||
logger.debug(f"Found {pkg}=={version} in {USER_ENV_PREFIX}")
|
||||
if V(version) < V(min_version):
|
||||
logger.info(
|
||||
f"{USER_ENV_PREFIX} has {pkg}=={version}, it will be upgraded to {pkg}>={min_version}"
|
||||
)
|
||||
to_upgrade.append(pkg)
|
||||
|
||||
if to_upgrade:
|
||||
conda.ensure_conda_packages(
|
||||
USER_ENV_PREFIX,
|
||||
# we _could_ explicitly pin Python here,
|
||||
# but conda already does this by default
|
||||
to_upgrade,
|
||||
)
|
||||
|
||||
conda.ensure_pip_requirements(
|
||||
USER_ENV_PREFIX,
|
||||
|
||||
Reference in New Issue
Block a user