From a659d1b48a40a08a67389abf08a6b7451adc0d99 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 18 Sep 2019 16:36:52 +0300 Subject: [PATCH 01/10] Make Python3.7 default and upgrade conda --- bootstrap/bootstrap.py | 15 ++++++++++----- tljh/installer.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 98e7245..5cdc17e 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -113,7 +113,11 @@ def main(): logger.setLevel(logging.DEBUG) logger.info('Checking if TLJH is already installed...') + python3_version = "python3" if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): + # Check if the existing install is under Python3.7 + if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3.7')): + python3_version = "python3.7" logger.info('TLJH already installed, upgrading...') initial_setup = False else: @@ -130,14 +134,15 @@ def main(): run_subprocess(['apt-get', 'update', '--yes']) run_subprocess(['apt-get', 'install', '--yes', - 'python3', - 'python3-venv', + 'python3.7', + 'python3.7-venv', + 'python3.7-dev', 'python3-pip', 'git' ]) logger.info('Installed python & virtual environment') os.makedirs(hub_prefix, exist_ok=True) - run_subprocess(['python3', '-m', 'venv', hub_prefix]) + run_subprocess(['python3.7', '-m', 'venv', hub_prefix]) logger.info('Set up hub virtual environment') if initial_setup: @@ -161,9 +166,9 @@ def main(): logger.info('Starting TLJH installer...') os.execv( - os.path.join(hub_prefix, 'bin', 'python3'), + os.path.join(hub_prefix, 'bin', python3_version), [ - os.path.join(hub_prefix, 'bin', 'python3'), + os.path.join(hub_prefix, 'bin', python3_version), '-m', 'tljh.installer', ] + sys.argv[1:] diff --git a/tljh/installer.py b/tljh/installer.py index c02d198..4fea072 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -237,8 +237,14 @@ def ensure_user_environment(user_requirements_txt_file): Set up user conda environment with required packages """ logger.info("Setting up user environment...") - miniconda_version = '4.5.4' - miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" + if "python3.7" in sys.executable: + conda_version = '4.7.5' + miniconda_version = '4.7.5' + miniconda_installer_md5 = "fc3595e7a66cf159a9d288a212834127" + else: + conda_version = '4.5.8' + miniconda_version = '4.5.4' + miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): logger.info('Downloading & setting up user environment...') @@ -247,7 +253,7 @@ def ensure_user_environment(user_requirements_txt_file): conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. - 'conda==4.5.8' + 'conda==' + conda_version ]) conda.ensure_pip_packages(USER_ENV_PREFIX, [ From fd81e9fef4538ae1a55d8f497a79611b5604a851 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 18 Sep 2019 17:57:56 +0300 Subject: [PATCH 02/10] Bump conda to 4.7.10 --- tests/test_conda.py | 4 ++-- tljh/installer.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index b1f8872..5d0f242 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -13,8 +13,8 @@ def prefix(): """ Provide a temporary directory with a conda environment """ - miniconda_version = '4.5.4' - miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" + miniconda_version = '4.7.10' + miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" with tempfile.TemporaryDirectory() as tmpdir: with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, tmpdir) diff --git a/tljh/installer.py b/tljh/installer.py index 4fea072..a136f90 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -238,9 +238,9 @@ def ensure_user_environment(user_requirements_txt_file): """ logger.info("Setting up user environment...") if "python3.7" in sys.executable: - conda_version = '4.7.5' - miniconda_version = '4.7.5' - miniconda_installer_md5 = "fc3595e7a66cf159a9d288a212834127" + conda_version = '4.7.10' + miniconda_version = '4.7.10' + miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" else: conda_version = '4.5.8' miniconda_version = '4.5.4' From 8b5d7ee79a5c75c1c462da6daa60ad166a1e18d7 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Wed, 22 Jan 2020 14:19:36 +0200 Subject: [PATCH 03/10] Bump conda in tests too --- tests/test_conda.py | 2 +- tljh/installer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index 5d0f242..95c7795 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -19,7 +19,7 @@ def prefix(): with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, tmpdir) conda.ensure_conda_packages(tmpdir, [ - 'conda==4.5.8' + 'conda==4.8.1' ]) yield tmpdir diff --git a/tljh/installer.py b/tljh/installer.py index a136f90..438ed08 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -238,7 +238,7 @@ def ensure_user_environment(user_requirements_txt_file): """ logger.info("Setting up user environment...") if "python3.7" in sys.executable: - conda_version = '4.7.10' + conda_version = '4.8.1' miniconda_version = '4.7.10' miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" else: From 6b930b24016fedaa0c9a1f9297d957c2670146fe Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 12:03:35 +0200 Subject: [PATCH 04/10] Don't upgrade venv python --- bootstrap/bootstrap.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 5cdc17e..98e7245 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -113,11 +113,7 @@ def main(): logger.setLevel(logging.DEBUG) logger.info('Checking if TLJH is already installed...') - python3_version = "python3" if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3')): - # Check if the existing install is under Python3.7 - if os.path.exists(os.path.join(hub_prefix, 'bin', 'python3.7')): - python3_version = "python3.7" logger.info('TLJH already installed, upgrading...') initial_setup = False else: @@ -134,15 +130,14 @@ def main(): run_subprocess(['apt-get', 'update', '--yes']) run_subprocess(['apt-get', 'install', '--yes', - 'python3.7', - 'python3.7-venv', - 'python3.7-dev', + 'python3', + 'python3-venv', 'python3-pip', 'git' ]) logger.info('Installed python & virtual environment') os.makedirs(hub_prefix, exist_ok=True) - run_subprocess(['python3.7', '-m', 'venv', hub_prefix]) + run_subprocess(['python3', '-m', 'venv', hub_prefix]) logger.info('Set up hub virtual environment') if initial_setup: @@ -166,9 +161,9 @@ def main(): logger.info('Starting TLJH installer...') os.execv( - os.path.join(hub_prefix, 'bin', python3_version), + os.path.join(hub_prefix, 'bin', 'python3'), [ - os.path.join(hub_prefix, 'bin', python3_version), + os.path.join(hub_prefix, 'bin', 'python3'), '-m', 'tljh.installer', ] + sys.argv[1:] From b645e6e933dba673aaa934a6e1434e25361d8f91 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 12:04:15 +0200 Subject: [PATCH 05/10] Only upgrade miniconda in fresh installs --- tljh/installer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index 438ed08..8497f24 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -237,16 +237,16 @@ def ensure_user_environment(user_requirements_txt_file): Set up user conda environment with required packages """ logger.info("Setting up user environment...") - if "python3.7" in sys.executable: - conda_version = '4.8.1' + + miniconda_old_version = '4.5.4' + conda_version = '4.5.8' + + # If no prior miniconda installation is found, we can install a newer version + if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version): miniconda_version = '4.7.10' miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" - else: - conda_version = '4.5.8' - miniconda_version = '4.5.4' - miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" + conda_version = '4.8.1' - if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): logger.info('Downloading & setting up user environment...') with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) From e6d340eb449a7fc5b083330083810ee2919cb12e Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 19:50:20 +0200 Subject: [PATCH 06/10] Document how to upgrade python for existing installs --- docs/howto/env/user-environment.rst | 54 +++++++++++++++++++++++++++++ docs/install/index.rst | 2 ++ 2 files changed, 56 insertions(+) diff --git a/docs/howto/env/user-environment.rst b/docs/howto/env/user-environment.rst index c46b08c..66c0144 100644 --- a/docs/howto/env/user-environment.rst +++ b/docs/howto/env/user-environment.rst @@ -153,3 +153,57 @@ The most common & portable way to fix this when using ``ssh`` is: .. code-block:: bash sudo PATH=${PATH} conda install -c conda-forge gdal + + +Upgrade to a newer Python version +================================= + +All new TLJH installs use miniconda 4.7.10, which comes with a Python 3.7 +environment for the users. The previously TLJH installs came with miniconda 4.5.4, +which meant a Python 3.6 environment. + +To upgrade the Python version of the user environment, one can: + +* **Start fresh on a machine that doesn't have TLJH already installed.** + + See the :ref:`installation guide ` section about how to install TLJH. + +* **Upgrade Python manually.** + + Because upgrading Python for existing installs can break packages alaredy installed + under the old Python, upgrading your current TLJH installation, will NOT upgrade + the Python version of the user environment, but you may do so manually. + + **Steps:** + + 1. Activate the user environment, if using ssh. + If the terminal was started with JupyterHub, this step can be skipped: + + .. code-block:: console + + source /opt/tljh/user/bin/activate + + 2. Get the list of currently installed pip packages (so you can later install them under the + new Python): + + .. code-block:: console + + pip freeze > pip_pkgs.txt + + 3. Update all conda installed packages in the environment: + + .. code-block:: console + + sudo conda update --all + + 4. Update Python version: + + .. code-block:: console + + conda install python=3.7 + + 5. Install the pip packages previously saved: + + .. code-block:: console + + pip install pip_pkgs.txt diff --git a/docs/install/index.rst b/docs/install/index.rst index a28baca..b783486 100644 --- a/docs/install/index.rst +++ b/docs/install/index.rst @@ -1,3 +1,5 @@ +.. _install/installing: + ========== Installing ========== From 1cc29df1fdeb1063a7a7d31b4aa846aa4978545b Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 19:53:58 +0200 Subject: [PATCH 07/10] Preserve PATH when using sudo --- docs/howto/env/user-environment.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/howto/env/user-environment.rst b/docs/howto/env/user-environment.rst index 66c0144..72ef43a 100644 --- a/docs/howto/env/user-environment.rst +++ b/docs/howto/env/user-environment.rst @@ -179,31 +179,31 @@ To upgrade the Python version of the user environment, one can: 1. Activate the user environment, if using ssh. If the terminal was started with JupyterHub, this step can be skipped: - .. code-block:: console + .. code-block:: bash source /opt/tljh/user/bin/activate 2. Get the list of currently installed pip packages (so you can later install them under the new Python): - .. code-block:: console + .. code-block:: bash pip freeze > pip_pkgs.txt 3. Update all conda installed packages in the environment: - .. code-block:: console + .. code-block:: bash - sudo conda update --all + sudo PATH=${PATH} conda update --all 4. Update Python version: - .. code-block:: console + .. code-block:: bash - conda install python=3.7 + sudo PATH=${PATH} conda install python=3.7 5. Install the pip packages previously saved: - .. code-block:: console + .. code-block:: bash pip install pip_pkgs.txt From 4aef4b3de70421e43f619c9dc11ec74c540271d6 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 19:59:44 +0200 Subject: [PATCH 08/10] Only install miniconda if no prev installs exist --- tljh/installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tljh/installer.py b/tljh/installer.py index 8497f24..f4d6a1e 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -242,7 +242,8 @@ def ensure_user_environment(user_requirements_txt_file): conda_version = '4.5.8' # If no prior miniconda installation is found, we can install a newer version - if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version): + if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version) and + not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): miniconda_version = '4.7.10' miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" conda_version = '4.8.1' From 063f92a9a6a5545d2be8c538ceed853b4c510677 Mon Sep 17 00:00:00 2001 From: GeorgianaElena Date: Thu, 30 Jan 2020 20:24:06 +0200 Subject: [PATCH 09/10] Refactor --- tljh/installer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index f4d6a1e..0e8b2ac 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -239,17 +239,18 @@ def ensure_user_environment(user_requirements_txt_file): logger.info("Setting up user environment...") miniconda_old_version = '4.5.4' + miniconda_new_version = '4.7.10' conda_version = '4.5.8' # If no prior miniconda installation is found, we can install a newer version - if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_old_version) and - not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): - miniconda_version = '4.7.10' + if not conda.check_miniconda_version( + USER_ENV_PREFIX, miniconda_old_version + ) and not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_new_version): miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" conda_version = '4.8.1' logger.info('Downloading & setting up user environment...') - with conda.download_miniconda_installer(miniconda_version, miniconda_installer_md5) as installer_path: + with conda.download_miniconda_installer(miniconda_new_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda.ensure_conda_packages(USER_ENV_PREFIX, [ From da9e7d86fc95f36a6ab493a4bdfac79d80c9feec Mon Sep 17 00:00:00 2001 From: Georgiana Elena Date: Fri, 31 Jan 2020 09:22:57 +0200 Subject: [PATCH 10/10] Update docs/howto/env/user-environment.rst Co-Authored-By: Yuvi Panda --- docs/howto/env/user-environment.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/env/user-environment.rst b/docs/howto/env/user-environment.rst index 72ef43a..5ea9fcb 100644 --- a/docs/howto/env/user-environment.rst +++ b/docs/howto/env/user-environment.rst @@ -206,4 +206,4 @@ To upgrade the Python version of the user environment, one can: .. code-block:: bash - pip install pip_pkgs.txt + pip install -r pip_pkgs.txt