Merge pull request #890 from consideRatio/pr/only-upgrade-whats-unconditionally-required

Only upgrade jupyterhub in user env when upgrading tljh, ensure pip>=23.1.2 in user env

.github/workflows/unit-test.yaml

@@ -81,10 +81,7 @@ jobs:
             ${{ hashFiles('setup.py', 'dev-requirements.txt', '.github/workflows/unit-test.yaml') }}
 
       - name: Install Python dependencies
-        # Keep pip version pinning in sync with the one in bootstrap.py!
-        # See changelog at https://pip.pypa.io/en/latest/news/#changelog
         run: |
-          python3 -m pip install -U "pip==23.1.*"
           python3 -m pip install -r dev-requirements.txt
           python3 -m pip install -e .
           pip freeze

MANIFEST.in

@@ -1,3 +1,3 @@
 include tljh/systemd-units/*
 include tljh/*.tpl
-include tljh/requirements-base.txt
+include tljh/requirements-*.txt

bootstrap/bootstrap.py

@@ -475,11 +475,8 @@ def main():
         os.makedirs(hub_env_prefix, exist_ok=True)
         run_subprocess(["python3", "-m", "venv", hub_env_prefix])
 
-    # Upgrade pip
-    # Keep pip version pinning in sync with the one in unit-test.yml!
-    # See changelog at https://pip.pypa.io/en/latest/news/#changelog
     logger.info("Upgrading pip...")
-    run_subprocess([hub_env_pip, "install", "--upgrade", "pip==23.1.*"])
+    run_subprocess([hub_env_pip, "install", "--upgrade", "pip"])
 
     # Install/upgrade TLJH installer
     tljh_install_cmd = [hub_env_pip, "install", "--upgrade"]

tljh/installer.py

@@ -106,25 +106,13 @@ def ensure_jupyterhub_package(prefix):
     hub environment be installed with pip prevents accidental mixing of python
     and conda packages!
     """
-    # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically
+    # Install dependencies for installing pycurl via pip, where build-essential
-    # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289
+    # is generally useful for installing other packages as well.
-    # build-essential is also generally useful to everyone involved, and required for pycurl
     apt.install_packages(["libssl-dev", "libcurl4-openssl-dev", "build-essential"])
-    conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True)
 
-    conda.ensure_pip_packages(
+    conda.ensure_pip_requirements(
         prefix,
-        [
+        os.path.join(HERE, "requirements-hub-env.txt"),
-            "jupyterhub==4.*",
-            "jupyterhub-systemdspawner==0.17.*",
-            "jupyterhub-firstuseauthenticator==1.*",
-            "jupyterhub-nativeauthenticator==1.*",
-            "jupyterhub-ldapauthenticator==1.*",
-            "jupyterhub-tmpauthenticator==0.6.*",
-            "oauthenticator==15.*",
-            "jupyterhub-idle-culler==1.*",
-            "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba",
-        ],
         upgrade=True,
     )
     traefik.ensure_traefik_binary(prefix)
@@ -158,9 +146,10 @@ MAMBAFORGE_CHECKSUMS = {
 
 # minimum versions of packages
 MINIMUM_VERSIONS = {
-    # if conda/mamba are lower than this, upgrade them before installing the user packages
+    # if conda/mamba/pip are lower than this, upgrade them before installing the user packages
     "mamba": "0.16.0",
     "conda": "4.10",
+    "pip": "23.1.2",
     # minimum Python version (if not matched, abort to avoid big disruptive updates)
     "python": "3.9",
 }
@@ -197,18 +186,16 @@ def ensure_user_environment(user_requirements_txt_file):
 
     # Check the existing environment for what to do
     package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
+    is_fresh_install = not package_versions
 
-    # Case 1: no existing environment
+    if is_fresh_install:
-    if not package_versions:
+        # If no Python environment is detected but a folder exists we abort to
-        # 1a. no environment, but prefix exists.
+        # avoid clobbering something we don't recognize.
-        # Abort to avoid clobbering something we don't recognize
         if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX):
             msg = f"Found non-empty directory that is not a conda install in {USER_ENV_PREFIX}. Please remove it (or rename it to preserve files) and run tljh again."
             logger.error(msg)
             raise OSError(msg)
 
-        # 1b. No environment, directory empty or doesn't exist
-        # start fresh install
         logger.info("Downloading & setting up user environment...")
         installer_url, installer_sha256 = _mambaforge_url()
         with conda.download_miniconda_installer(
@@ -216,11 +203,12 @@ def ensure_user_environment(user_requirements_txt_file):
         ) as installer_path:
             conda.install_miniconda(installer_path, USER_ENV_PREFIX)
         package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX)
+
         # quick sanity check: we should have conda and mamba!
         assert "conda" in package_versions
         assert "mamba" in package_versions
 
-    # next, check Python
+    # Check Python version
     python_version = package_versions["python"]
     logger.debug(f"Found python={python_version} in {USER_ENV_PREFIX}")
     if V(python_version) < V(MINIMUM_VERSIONS["python"]):
@@ -232,10 +220,14 @@ def ensure_user_environment(user_requirements_txt_file):
         logger.error(msg)
         raise ValueError(msg)
 
-    # at this point, we know we have an env ready with conda and are going to start installing
+    # Ensure minimum versions of the following packages by upgrading to the
-    # first, check if we should upgrade/install conda and/or mamba
+    # latest if below that version.
+    #
+    # - conda/mamba, via conda-forge
+    # - pip,         via PyPI
+    #
     to_upgrade = []
-    for pkg in ("conda", "mamba"):
+    for pkg in ("conda", "mamba", "pip"):
         version = package_versions.get(pkg)
         min_version = MINIMUM_VERSIONS[pkg]
         if not version:
@@ -249,19 +241,36 @@ def ensure_user_environment(user_requirements_txt_file):
                 )
                 to_upgrade.append(pkg)
 
-    if to_upgrade:
+        cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"})
-        conda.ensure_conda_packages(
+        if cf_pkgs_to_upgrade:
-            USER_ENV_PREFIX,
+            conda.ensure_conda_packages(
-            # we _could_ explicitly pin Python here,
+                USER_ENV_PREFIX,
-            # but conda already does this by default
+                # we _could_ explicitly pin Python here,
-            to_upgrade,
+                # but conda already does this by default
-        )
+                cf_pkgs_to_upgrade,
+            )
+        pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"})
+        if pypi_pkgs_to_upgrade:
+            conda.ensure_pip_packages(
+                USER_ENV_PREFIX,
+                pypi_pkgs_to_upgrade,
+                upgrade=True,
+            )
 
-    conda.ensure_pip_requirements(
+    # Install/upgrade the jupyterhub version in the user env based on the
-        USER_ENV_PREFIX,
+    # version specification used for the hub env.
-        os.path.join(HERE, "requirements-base.txt"),
+    #
-        upgrade=True,
+    with open(os.path.join(HERE, "requirements-hub-env.txt")) as f:
-    )
+        jh_version_spec = [l for l in f if l.startswith("jupyterhub>=")][0]
+    conda.ensure_pip_packages(USER_ENV_PREFIX, [jh_version_spec], upgrade=True)
+
+    # Install user environment extras for initial installations
+    #
+    if is_fresh_install:
+        conda.ensure_pip_requirements(
+            USER_ENV_PREFIX,
+            os.path.join(HERE, "requirements-user-env-extras.txt"),
+        )
 
     if user_requirements_txt_file:
         # FIXME: This currently fails hard, should fail soft and not abort installer

tljh/requirements-hub-env.txt

@@ -0,0 +1,29 @@
+# When tljh.installer runs, the hub' environment as typically found in
+# /opt/tljh/hub, is upgraded to use these packages.
+#
+# When a new release is made, the lower bounds should be incremented to the
+# latest release to help us narrow the versions based on knowing what tljh
+# version is installed from inspecting this file.
+#
+# If a dependency is bumped to a new major version, we should make a major
+# version release of tljh.
+#
+jupyterhub>=4.0.0,<5
+jupyterhub-systemdspawner>=0.17.0,<5
+jupyterhub-firstuseauthenticator>=1.0.0,<2
+jupyterhub-nativeauthenticator>=1.1.0,<2
+jupyterhub-ldapauthenticator>=1.3.2,<2
+jupyterhub-tmpauthenticator>=0.6.0,<0.7
+oauthenticator>=15.1.0,<16
+jupyterhub-idle-culler>=1.2.1,<2
+git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba
+
+# pycurl is installed to improve reliability and performance for when JupyterHub
+# makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when
+# making requests over tornado's SimpleHTTPClient automatically if pycurl is
+# installed.
+#
+# ref: https://www.tornadoweb.org/en/stable/httpclient.html#module-tornado.simple_httpclient
+# ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289
+#
+pycurl>=7.45.2,<8

tljh/requirements-user-env-extras.txt

@@ -1,14 +1,14 @@
 # When tljh.installer runs, the users' environment as typically found in
-# /opt/tljh/user, is setup with these packages.
+# /opt/tljh/user, is installed with these packages.
+#
+# Whats listed here represents additional packages that the distributions
+# installs initially, but doesn't upgrade as tljh is upgraded.
 #
 # WARNING: The order of these dependencies matters, this was observed when using
 #          the requirements-txt-fixer pre-commit hook that sorted them and made
 #          our integration tests fail.
 #
-# JupyterHub + notebook package are base requirements for user environment
-jupyterhub==4.*
 notebook==6.*
-# Install additional notebook frontends!
 jupyterlab==3.*
 # nbgitpuller for easily pulling in Git repositories
 nbgitpuller==1.*