zhangyiss/the-littlest-jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/the-littlest-jupyterhub.git synced 2025-12-18 21:54:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into conda-channels

Browse Source
This commit is contained in:
YuviPanda
2023-09-29 14:27:25 -07:00
parent ff1f612d10 44bdaf2fa4
commit 6359bf498c
180 changed files with 7796 additions and 6028 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
tljh/installer.py
View File

@@ -17,15 +17,8 @@ import pluggy
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from tljh import (
apt,
conda,
hooks,
migrator,
systemd,
traefik,
user,
)
from tljh import apt, conda, hooks, migrator, systemd, traefik, user
from .config import (
CONFIG_DIR,
CONFIG_FILE,
@@ -34,6 +27,7 @@ from .config import (
STATE_DIR,
USER_ENV_PREFIX,
)
from .utils import parse_version as V
from .yaml import yaml
HERE = os.path.abspath(os.path.dirname(__file__))
@@ -112,25 +106,13 @@ def ensure_jupyterhub_package(prefix):
hub environment be installed with pip prevents accidental mixing of python
and conda packages!
"""
# Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically
# pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289
# build-essential is also generally useful to everyone involved, and required for pycurl
# Install dependencies for installing pycurl via pip, where build-essential
# is generally useful for installing other packages as well.
apt.install_packages(["libssl-dev", "libcurl4-openssl-dev", "build-essential"])
conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True)
conda.ensure_pip_packages(
conda.ensure_pip_requirements(
prefix,
[
"jupyterhub==1.*",
"jupyterhub-systemdspawner==0.16.*",
"jupyterhub-firstuseauthenticator==1.*",
"jupyterhub-nativeauthenticator==1.*",
"jupyterhub-ldapauthenticator==1.*",
"jupyterhub-tmpauthenticator==0.6.*",
"oauthenticator==14.*",
"jupyterhub-idle-culler==1.*",
"git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba",
],
os.path.join(HERE, "requirements-hub-env.txt"),
upgrade=True,
)
traefik.ensure_traefik_binary(prefix)
@@ -144,6 +126,7 @@ def ensure_usergroups():
user.ensure_group("jupyterhub-users")
logger.info("Granting passwordless sudo to JupyterHub admins...")
os.makedirs("/etc/sudoers.d/", exist_ok=True)
with open("/etc/sudoers.d/jupyterhub-admins", "w") as f:
# JupyterHub admins should have full passwordless sudo access
f.write("%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n")
@@ -153,66 +136,163 @@ def ensure_usergroups():
f.write("Defaults exempt_group = jupyterhub-admins\n")
# Install mambaforge using an installer from
# https://github.com/conda-forge/miniforge/releases
MAMBAFORGE_VERSION = "23.1.0-1"
# sha256 checksums
MAMBAFORGE_CHECKSUMS = {
"aarch64": "d9d89c9e349369702171008d9ee7c5ce80ed420e5af60bd150a3db4bf674443a",
"x86_64": "cfb16c47dc2d115c8b114280aa605e322173f029fdb847a45348bf4bd23c62ab",
}
# minimum versions of packages
MINIMUM_VERSIONS = {
# if conda/mamba/pip are lower than this, upgrade them before installing the user packages
"mamba": "0.16.0",
"conda": "4.10",
"pip": "23.1.2",
# minimum Python version (if not matched, abort to avoid big disruptive updates)
"python": "3.9",
}
def _mambaforge_url(version=MAMBAFORGE_VERSION, arch=None):
"""Return (URL, checksum) for mambaforge download for a given version and arch
Default values provided for both version and arch
"""
if arch is None:
arch = os.uname().machine
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(
v=version,
arch=arch,
)
# Check system architecture, set appropriate installer checksum
checksum = MAMBAFORGE_CHECKSUMS.get(arch)
if not checksum:
raise ValueError(
f"Unsupported architecture: {arch}. TLJH only supports {','.join(MAMBAFORGE_CHECKSUMS.keys())}"
)
return installer_url, checksum
def ensure_user_environment(user_requirements_txt_file):
"""
Set up user conda environment with required packages
"""
logger.info("Setting up user environment...")
miniconda_old_version = "4.5.4"
miniconda_new_version = "4.7.10"
# Install mambaforge using an installer from
# https://github.com/conda-forge/miniforge/releases
mambaforge_new_version = "4.10.3-7"
# Check system architecture, set appropriate installer checksum
if os.uname().machine == "aarch64":
installer_sha256 = (
"ac95f137b287b3408e4f67f07a284357b1119ee157373b788b34e770ef2392b2"
)
elif os.uname().machine == "x86_64":
installer_sha256 = (
"fc872522ec427fcab10167a93e802efaf251024b58cc27b084b915a9a73c4474"
)
# Check OS, set appropriate string for conda installer path
if os.uname().sysname != "Linux":
raise OSError("TLJH is only supported on Linux platforms.")
# Then run `mamba --version` to get the conda and mamba versions
# Keep these in sync with tests/test_conda.py::prefix
mambaforge_conda_new_version = "4.10.3"
mambaforge_mamba_version = "0.16.0"
if conda.check_miniconda_version(USER_ENV_PREFIX, mambaforge_conda_new_version):
conda_version = "4.10.3"
elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version):
conda_version = "4.8.1"
elif conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version):
conda_version = "4.5.8"
# If no prior miniconda installation is found, we can install a newer version
else:
# Check the existing environment for what to do
package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
is_fresh_install = not package_versions
if is_fresh_install:
# If no Python environment is detected but a folder exists we abort to
# avoid clobbering something we don't recognize.
if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX):
msg = f"Found non-empty directory that is not a conda install in {USER_ENV_PREFIX}. Please remove it (or rename it to preserve files) and run tljh again."
logger.error(msg)
raise OSError(msg)
logger.info("Downloading & setting up user environment...")
installer_url = "https://github.com/conda-forge/miniforge/releases/download/{v}/Mambaforge-{v}-Linux-{arch}.sh".format(
v=mambaforge_new_version, arch=os.uname().machine
)
installer_url, installer_sha256 = _mambaforge_url()
with conda.download_miniconda_installer(
installer_url, installer_sha256
) as installer_path:
conda.install_miniconda(installer_path, USER_ENV_PREFIX)
conda_version = "4.10.3"
package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
conda.ensure_conda_packages(
USER_ENV_PREFIX,
[
# Conda's latest version is on conda much more so than on PyPI.
"conda==" + conda_version,
"mamba==" + mambaforge_mamba_version,
],
)
# quick sanity check: we should have conda and mamba!
assert "conda" in package_versions
assert "mamba" in package_versions
conda.ensure_pip_requirements(
USER_ENV_PREFIX,
os.path.join(HERE, "requirements-base.txt"),
upgrade=True,
)
# Check Python version
python_version = package_versions["python"]
logger.debug(f"Found python={python_version} in {USER_ENV_PREFIX}")
if V(python_version) < V(MINIMUM_VERSIONS["python"]):
msg = (
f"TLJH requires Python >={MINIMUM_VERSIONS['python']}, found python={python_version} in {USER_ENV_PREFIX}."
f"\nPlease upgrade Python (may be highly disruptive!), or remove/rename {USER_ENV_PREFIX} to allow TLJH to make a fresh install."
f"\nYou can use `{USER_ENV_PREFIX}/bin/conda list` to save your current list of packages."
)
logger.error(msg)
raise ValueError(msg)
# Ensure minimum versions of the following packages by upgrading to the
# latest if below that version.
#
# - conda/mamba, via conda-forge
# - pip, via PyPI
#
to_upgrade = []
for pkg in ("conda", "mamba", "pip"):
version = package_versions.get(pkg)
min_version = MINIMUM_VERSIONS[pkg]
if not version:
logger.warning(f"{USER_ENV_PREFIX} is missing {pkg}, installing it...")
to_upgrade.append(pkg)
else:
logger.debug(f"Found {pkg}=={version} in {USER_ENV_PREFIX}")
if V(version) < V(min_version):
logger.info(
f"{USER_ENV_PREFIX} has {pkg}=={version}, it will be upgraded to {pkg}>={min_version}"
)
to_upgrade.append(pkg)
# force reinstall conda/mamba to ensure a basically consistent env
# avoids issues with RemoveError: 'requests' is a dependency of conda
# only do this for 'old' conda versions known to have a problem
# we don't know how old, but we know 4.10 is affected and 23.1 is not
if not is_fresh_install and V(package_versions.get("conda", "0")) < V("23.1"):
# force-reinstall doesn't upgrade packages
# it reinstalls them in-place
# only reinstall packages already present
to_reinstall = []
for pkg in ["conda", "mamba"]:
if pkg in package_versions:
# add version pin to avoid upgrades
to_reinstall.append(f"{pkg}=={package_versions[pkg]}")
logger.info(
f"Reinstalling {', '.join(to_reinstall)} to ensure a consistent environment"
)
conda.ensure_conda_packages(
USER_ENV_PREFIX, list(to_reinstall), force_reinstall=True
)
cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"})
if cf_pkgs_to_upgrade:
conda.ensure_conda_packages(
USER_ENV_PREFIX,
# we _could_ explicitly pin Python here,
# but conda already does this by default
cf_pkgs_to_upgrade,
)
pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"})
if pypi_pkgs_to_upgrade:
conda.ensure_pip_packages(
USER_ENV_PREFIX,
pypi_pkgs_to_upgrade,
upgrade=True,
)
# Install/upgrade the jupyterhub version in the user env based on the
# version specification used for the hub env.
#
with open(os.path.join(HERE, "requirements-hub-env.txt")) as f:
jh_version_spec = [l for l in f if l.startswith("jupyterhub>=")][0]
conda.ensure_pip_packages(USER_ENV_PREFIX, [jh_version_spec], upgrade=True)
# Install user environment extras for initial installations
#
if is_fresh_install:
conda.ensure_pip_requirements(
USER_ENV_PREFIX,
os.path.join(HERE, "requirements-user-env-extras.txt"),
)
if user_requirements_txt_file:
# FIXME: This currently fails hard, should fail soft and not abort installer
@@ -225,7 +305,7 @@ def ensure_user_environment(user_requirements_txt_file):
def ensure_admins(admin_password_list):
"""
Setup given list of users as admins.
Setup given list of user[:password] strings as admins.
"""
os.makedirs(STATE_DIR, mode=0o700, exist_ok=True)
@@ -450,7 +530,7 @@ def main():
ensure_admins(args.admin)
ensure_usergroups()
if args.user_requirements_txt_url:
logger.info("installing packages from user_requirements_txt_url")
logger.info("installing packages from user_requirements_txt_url")
ensure_user_environment(args.user_requirements_txt_url)
logger.info("Setting up JupyterHub...")
@@ -464,7 +544,6 @@ def main():
print("Progress page server stopped successfully.")
except Exception as e:
logger.error(f"Couldn't stop the progress page server. Exception was {e}.")
pass
ensure_jupyterhub_service(HUB_ENV_PREFIX)
ensure_jupyterhub_running()