From fc8f70463c7b03ce8cb0e1fdceddb548d894ceeb Mon Sep 17 00:00:00 2001 From: Will Dampier Date: Fri, 2 Sep 2022 14:10:43 -0400 Subject: [PATCH 1/9] Adding the ability to define conda channels --- tests/test_conda.py | 9 +++++++++ tljh/conda.py | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_conda.py b/tests/test_conda.py index a13ab39..c49f126 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -43,6 +43,15 @@ def test_ensure_packages(prefix): # Throws an error if this fails subprocess.check_call([os.path.join(prefix, "bin", "python"), "-c", "import numpy"]) + +def test_ensure_channel_packages(prefix): + """ + Test installing packages in conda environment + """ + conda.ensure_conda_packages(prefix, ["csvtk"], channels=('conda-forge', 'bioconda')) + # Throws an error if this fails + subprocess.check_call([os.path.join(prefix, "bin", "csvtk"), "cat", "--help"]) + def test_ensure_pip_packages(prefix): """ diff --git a/tljh/conda.py b/tljh/conda.py index 88923f6..12ffe3e 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -90,9 +90,9 @@ def install_miniconda(installer_path, prefix): fix_permissions(prefix) -def ensure_conda_packages(prefix, packages): +def ensure_conda_packages(prefix, packages, channels=('conda-forge',)): """ - Ensure packages (from conda-forge) are installed in the conda prefix. + Ensure packages (from channels) are installed in the conda prefix. Note that conda seem to update dependencies by default, so there is probably no need to have a update parameter exposed for this function. @@ -103,16 +103,18 @@ def ensure_conda_packages(prefix, packages): # Explicitly do *not* capture stderr, since that's not always JSON! # Scripting conda is a PITA! # FIXME: raise different exception when using + + channel_cmd = '-c ' + ' -c '.join(channels) + raw_output = subprocess.check_output( conda_executable + [ "install", - "-c", - "conda-forge", # Make customizable if we ever need to "--json", "--prefix", abspath, ] + + channel_cmd.split() + packages ).decode() # `conda install` outputs JSON lines for fetch updates, From 8d1033393c134a618a105b0589c7b641dab863ba Mon Sep 17 00:00:00 2001 From: Will Dampier Date: Fri, 2 Sep 2022 14:46:55 -0400 Subject: [PATCH 2/9] adding tljh_extra_user_conda_channels hook to the hooks and installer --- tljh/hooks.py | 8 ++++++++ tljh/installer.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tljh/hooks.py b/tljh/hooks.py index ddb1f3f..0a94a62 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -15,6 +15,14 @@ def tljh_extra_user_conda_packages(): pass +@hookspec +def tljh_extra_user_conda_channels(): + """ + Return a list of conda channels to be used during user environment installation. + """ + pass + + @hookspec def tljh_extra_user_pip_packages(): """ diff --git a/tljh/installer.py b/tljh/installer.py index 8da932c..baef3ef 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -370,13 +370,16 @@ def run_plugin_actions(plugin_manager): # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) + conda_channels = list(itertools.chain(*hook.tljh_extra_user_conda_channels())) + if len(conda_channels) == 0: + conda_channels = ('conda-forge',) if conda_packages: logger.info( "Installing {} user conda packages collected from plugins: {}".format( len(conda_packages), " ".join(conda_packages) ) ) - conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) + conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages, conda_channels=conda_channels) # Install pip packages user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) From 58181c967170f685e61a15034326b47eb313f4ba Mon Sep 17 00:00:00 2001 From: Will Dampier Date: Fri, 2 Sep 2022 14:47:25 -0400 Subject: [PATCH 3/9] adjusted integration test to call new hook --- integration-tests/plugins/simplest/tljh_simplest.py | 8 ++++++++ integration-tests/test_simplest_plugin.py | 1 + 2 files changed, 9 insertions(+) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index a134083..12a64b4 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -8,6 +8,14 @@ from tljh.hooks import hookimpl def tljh_extra_user_conda_packages(): return [ "hypothesis", + "csvtk" + ] + +@hookimpl +def tljh_extra_user_conda_channels(): + return [ + "conda-forge", + "bioconda" ] diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index eebff27..e5a3b04 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -33,6 +33,7 @@ def test_conda_packages(): Test extra user conda packages are installed """ subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import hypothesis"]) + subprocess.check_call([f"{USER_ENV_PREFIX}/bin/csvtk", "cat", "--help"]) def test_config_hook(): From ff1f612d10df742c09ad700ed453c4760a41473f Mon Sep 17 00:00:00 2001 From: Will Dampier Date: Fri, 2 Sep 2022 14:47:41 -0400 Subject: [PATCH 4/9] adding content to docs --- docs/contributing/plugins.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/contributing/plugins.rst b/docs/contributing/plugins.rst index ecbf398..d718b60 100644 --- a/docs/contributing/plugins.rst +++ b/docs/contributing/plugins.rst @@ -114,6 +114,22 @@ environment from conda-forge. 'iris', 'dask', ] + +By default packages are only installed from the ``conda-forge`` channel. +If you need other channels, like ``bioconda`` or self-made channels, +they can be added by using the ``tljh_extra_user_conda_channels`` hook. + +.. code-block:: python + + from tljh.hooks import hookimpl + + @hookimpl + def tljh_extra_user_conda_channels(): + return [ + 'conda-forge', + 'bioconda', + 'fastai', + ] Publishing plugins From 9bde7e4680acc9a18ae660a99acf0030320594a6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 21:35:50 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- integration-tests/plugins/simplest/tljh_simplest.py | 12 +++--------- integration-tests/test_simplest_plugin.py | 1 + tests/test_conda.py | 6 +++--- tljh/hooks.py | 1 - tljh/installer.py | 6 ++++-- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index d122acd..8b663b3 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -6,18 +6,12 @@ from tljh.hooks import hookimpl @hookimpl def tljh_extra_user_conda_packages(): - return [ - "hypothesis", - "csvtk", - "tqdm" - ] + return ["hypothesis", "csvtk", "tqdm"] + @hookimpl def tljh_extra_user_conda_channels(): - return [ - "conda-forge", - "bioconda" - ] + return ["conda-forge", "bioconda"] @hookimpl diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index 68bd859..055c9b1 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -33,6 +33,7 @@ def test_conda_packages(): subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import hypothesis"]) subprocess.check_call([f"{USER_ENV_PREFIX}/bin/csvtk", "cat", "--help"]) + def test_tljh_extra_apt_packages(): assert os.path.exists("/usr/games/sl") diff --git a/tests/test_conda.py b/tests/test_conda.py index 575b831..de69f71 100644 --- a/tests/test_conda.py +++ b/tests/test_conda.py @@ -32,15 +32,15 @@ def test_ensure_packages(prefix): # Throws an error if this fails subprocess.check_call([os.path.join(prefix, "bin", "python"), "-c", "import numpy"]) - + def test_ensure_channel_packages(prefix): """ Test installing packages in conda environment """ - conda.ensure_conda_packages(prefix, ["csvtk"], channels=('conda-forge', 'bioconda')) + conda.ensure_conda_packages(prefix, ["csvtk"], channels=("conda-forge", "bioconda")) # Throws an error if this fails subprocess.check_call([os.path.join(prefix, "bin", "csvtk"), "cat", "--help"]) - + def test_ensure_pip_packages(prefix): """ diff --git a/tljh/hooks.py b/tljh/hooks.py index ce341fd..151134d 100644 --- a/tljh/hooks.py +++ b/tljh/hooks.py @@ -19,7 +19,6 @@ def tljh_extra_user_conda_channels(): """ Return a list of conda channels to be used during user environment installation. """ - pass @hookspec diff --git a/tljh/installer.py b/tljh/installer.py index f2b68c7..3e7c3e4 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -452,14 +452,16 @@ def run_plugin_actions(plugin_manager): conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) conda_channels = list(itertools.chain(*hook.tljh_extra_user_conda_channels())) if len(conda_channels) == 0: - conda_channels = ('conda-forge',) + conda_channels = ("conda-forge",) if conda_packages: logger.info( "Installing {} user conda packages collected from plugins: {}".format( len(conda_packages), " ".join(conda_packages) ) ) - conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages, conda_channels=conda_channels) + conda.ensure_conda_packages( + USER_ENV_PREFIX, conda_packages, conda_channels=conda_channels + ) # Install pip packages user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) From 58a679f584d40dd531731b30ed2a07414a548d2f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 29 Sep 2023 14:38:54 -0700 Subject: [PATCH 6/9] Fix how channels are added to -c command --- tljh/conda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tljh/conda.py b/tljh/conda.py index aa4b051..c39dc32 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -117,7 +117,8 @@ def ensure_conda_packages(prefix, packages, channels=('conda-forge',), force_rei # avoids problems with RemoveError upgrading conda from old versions cmd += ["--force-reinstall"] - cmd += ["-c", channel for channel in channels] + for channel in channels: + cmd += ["-c", channel] abspath = os.path.abspath(prefix) From eafc10e82af9eea42d0c0bc540f4ca50d4db0c99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 21:39:30 +0000 Subject: [PATCH 7/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tljh/conda.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tljh/conda.py b/tljh/conda.py index c39dc32..6b958ef 100644 --- a/tljh/conda.py +++ b/tljh/conda.py @@ -98,7 +98,9 @@ def install_miniconda(installer_path, prefix): fix_permissions(prefix) -def ensure_conda_packages(prefix, packages, channels=('conda-forge',), force_reinstall=False): +def ensure_conda_packages( + prefix, packages, channels=("conda-forge",), force_reinstall=False +): """ Ensure packages (from channels) are installed in the conda prefix. From 291c096a1731cba172ec2e5c48bb1db25edededd Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 29 Sep 2023 14:46:57 -0700 Subject: [PATCH 8/9] Fix argument name of channels --- tljh/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/installer.py b/tljh/installer.py index 3e7c3e4..95557fc 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -460,7 +460,7 @@ def run_plugin_actions(plugin_manager): ) ) conda.ensure_conda_packages( - USER_ENV_PREFIX, conda_packages, conda_channels=conda_channels + USER_ENV_PREFIX, conda_packages, channels=conda_channels ) # Install pip packages From 9111b73cee33d31735376d4110595e763469110f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 4 Sep 2024 16:04:25 +0200 Subject: [PATCH 9/9] tests: reduce a test to only one package per conda channel --- integration-tests/plugins/simplest/tljh_simplest.py | 4 +++- integration-tests/test_simplest_plugin.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/integration-tests/plugins/simplest/tljh_simplest.py b/integration-tests/plugins/simplest/tljh_simplest.py index c3f978d..f5ceca4 100644 --- a/integration-tests/plugins/simplest/tljh_simplest.py +++ b/integration-tests/plugins/simplest/tljh_simplest.py @@ -7,7 +7,9 @@ from tljh.hooks import hookimpl @hookimpl def tljh_extra_user_conda_packages(): - return ["hypothesis", "csvtk", "tqdm"] + # tqdm installs from the conda-forge channel (https://conda-forge.org/packages/) + # csvtk installs from the bioconda channel (https://bioconda.github.io/conda-package_index.html) + return ["tqdm", "csvtk"] @hookimpl diff --git a/integration-tests/test_simplest_plugin.py b/integration-tests/test_simplest_plugin.py index ce01d0e..2c65049 100644 --- a/integration-tests/test_simplest_plugin.py +++ b/integration-tests/test_simplest_plugin.py @@ -29,9 +29,12 @@ def test_tljh_extra_hub_pip_packages(): def test_conda_packages(): """ - Test extra user conda packages are installed + Test extra user conda packages are installed from multiple channels. + + - tqdm installs from the conda-forge channel (https://conda-forge.org/packages/) + - csvtk installs from the bioconda channel (https://bioconda.github.io/conda-package_index.html) """ - subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import hypothesis"]) + subprocess.check_call([f"{USER_ENV_PREFIX}/bin/python3", "-c", "import tqdm"]) subprocess.check_call([f"{USER_ENV_PREFIX}/bin/csvtk", "cat", "--help"])