Merge pull request #920 from minrk/upgrade-mamba-conda

--force-reinstall conda to ensure it's working before we try to install conda packages

tljh/conda.py

@@ -98,7 +98,7 @@ def install_miniconda(installer_path, prefix):
     fix_permissions(prefix)
 
 
-def ensure_conda_packages(prefix, packages):
+def ensure_conda_packages(prefix, packages, force_reinstall=False):
     """
     Ensure packages (from conda-forge) are installed in the conda prefix.
 
@@ -110,13 +110,18 @@ def ensure_conda_packages(prefix, packages):
         # fallback on conda if mamba is not present (e.g. for mamba to install itself)
         conda_executable = os.path.join(prefix, "bin", "conda")
 
+    cmd = [conda_executable, "install", "--yes"]
+
+    if force_reinstall:
+        # use force-reinstall, e.g. for conda/mamba to ensure everything is okay
+        # avoids problems with RemoveError upgrading conda from old versions
+        cmd += ["--force-reinstall"]
+
     abspath = os.path.abspath(prefix)
 
     utils.run_subprocess(
-        [
+        cmd
-            conda_executable,
+        + [
-            "install",
-            "-y",
             "-c",
             "conda-forge",  # Make customizable if we ever need to
             "--prefix",

tljh/installer.py

@@ -242,21 +242,42 @@ def ensure_user_environment(user_requirements_txt_file):
                 )
                 to_upgrade.append(pkg)
 
-        cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"})
+    # force reinstall conda/mamba to ensure a basically consistent env
-        if cf_pkgs_to_upgrade:
+    # avoids issues with RemoveError: 'requests' is a dependency of conda
-            conda.ensure_conda_packages(
+    # only do this for 'old' conda versions known to have a problem
-                USER_ENV_PREFIX,
+    # we don't know how old, but we know 4.10 is affected and 23.1 is not
-                # we _could_ explicitly pin Python here,
+    if not is_fresh_install and V(package_versions.get("conda", "0")) < V("23.1"):
-                # but conda already does this by default
+        # force-reinstall doesn't upgrade packages
-                cf_pkgs_to_upgrade,
+        # it reinstalls them in-place
-            )
+        # only reinstall packages already present
-        pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"})
+        to_reinstall = []
-        if pypi_pkgs_to_upgrade:
+        for pkg in ["conda", "mamba"]:
-            conda.ensure_pip_packages(
+            if pkg in package_versions:
-                USER_ENV_PREFIX,
+                # add version pin to avoid upgrades
-                pypi_pkgs_to_upgrade,
+                to_reinstall.append(f"{pkg}=={package_versions[pkg]}")
-                upgrade=True,
+        logger.info(
-            )
+            f"Reinstalling {', '.join(to_reinstall)} to ensure a consistent environment"
+        )
+        conda.ensure_conda_packages(
+            USER_ENV_PREFIX, list(to_reinstall), force_reinstall=True
+        )
+
+    cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"})
+    if cf_pkgs_to_upgrade:
+        conda.ensure_conda_packages(
+            USER_ENV_PREFIX,
+            # we _could_ explicitly pin Python here,
+            # but conda already does this by default
+            cf_pkgs_to_upgrade,
+        )
+
+    pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"})
+    if pypi_pkgs_to_upgrade:
+        conda.ensure_pip_packages(
+            USER_ENV_PREFIX,
+            pypi_pkgs_to_upgrade,
+            upgrade=True,
+        )
 
     # Install/upgrade the jupyterhub version in the user env based on the
     # version specification used for the hub env.