From 94b38def7d9f59dbf4f2302317541ca59a9f10d2 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 29 Dec 2020 16:22:57 +0530 Subject: [PATCH 1/5] Upgrade to JupyterLab 3.0 JupyterLab 3.0 no longer equires npm to build extensions! This should hopefully let us reduce the memory requiremets for TLJH. --- integration-tests/test_extensions.py | 2 +- tljh/installer.py | 31 ---------------------------- tljh/requirements-base.txt | 4 +++- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/integration-tests/test_extensions.py b/integration-tests/test_extensions.py index 319dc4b..a56bd16 100644 --- a/integration-tests/test_extensions.py +++ b/integration-tests/test_extensions.py @@ -13,7 +13,7 @@ def test_serverextensions(): ], stderr=subprocess.PIPE) extensions = [ - 'jupyterlab 2.', + 'jupyterlab 3.', 'nbgitpuller 0.9.', 'nteract_on_jupyter 2.1.', 'nbresuse ' diff --git a/tljh/installer.py b/tljh/installer.py index f3cc989..5852963 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -166,36 +166,6 @@ def ensure_jupyterhub_service(prefix): systemd.enable_service('traefik') -def ensure_jupyterlab_extensions(): - """ - Install the JupyterLab extensions we want. - """ - extensions = [ - # We don't pin versions here, since labextension will find something - # appropriate for our version of jupyterlab - '@jupyter-widgets/jupyterlab-manager' - ] - install_options = [ - '--no-build' # do not build extension at install time. Will build later - ] - utils.run_subprocess([ - os.path.join(USER_ENV_PREFIX, 'bin/jupyter'), - 'labextension', - 'install' - ] + extensions + install_options) - - # Build all the lab extensions in one go using jupyter lab build command - build_options = [ - '--minimize=False', - '--dev-build=False' - ] - - utils.run_subprocess([ - os.path.join(USER_ENV_PREFIX, 'bin/jupyter'), - 'lab', - 'build' - ] + build_options) - def ensure_jupyterhub_package(prefix): """ @@ -508,7 +478,6 @@ def main(): logger.info("Setting up JupyterHub...") ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) - ensure_jupyterlab_extensions() # Stop the http server with the progress page before traefik starts if args.progress_page_server_pid: diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 5681d60..92be176 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -5,8 +5,10 @@ jupyterhub==1.2.* notebook==6.1.* # Install additional notebook frontends! -jupyterlab==2.* +jupyterlab==3.* nteract-on-jupyter==2.1.* +# Install jupyterlab extensions from PyPI +jupyterlab-widgets==1.* # nbgitpuller for easily pulling in Git repositories nbgitpuller==0.9.* # nbresuse to show people how much RAM they are using From 0b5f9c6788cbf84cb4ab16e8fad0d513cfefe48b Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Mon, 8 Feb 2021 17:54:47 +0100 Subject: [PATCH 2/5] Update base requirements --- tljh/requirements-base.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index 92be176..e4a6e97 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -8,12 +8,11 @@ notebook==6.1.* jupyterlab==3.* nteract-on-jupyter==2.1.* # Install jupyterlab extensions from PyPI -jupyterlab-widgets==1.* # nbgitpuller for easily pulling in Git repositories nbgitpuller==0.9.* # nbresuse to show people how much RAM they are using nbresuse==0.3.* # Most people consider ipywidgets to be part of the core notebook experience -ipywidgets==7.5.* +ipywidgets==7.6.* # Pin tornado -tornado<6 +tornado>=6.1 From 873e6bf661d8cc5cd855b3a087636221a96a755d Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Mon, 8 Feb 2021 18:01:28 +0100 Subject: [PATCH 3/5] Remove Node.js --- docs/contributing/packages.rst | 4 +-- docs/topic/installer-actions.rst | 10 +++--- tests/test_installer.py | 5 --- tljh/installer.py | 62 -------------------------------- 4 files changed, 5 insertions(+), 76 deletions(-) diff --git a/docs/contributing/packages.rst b/docs/contributing/packages.rst index 484b0d1..74613e2 100644 --- a/docs/contributing/packages.rst +++ b/docs/contributing/packages.rst @@ -40,9 +40,7 @@ Apt packages ============ Base operating system packages, including Python itself, are installed -via ``apt`` from the base Ubuntu repositories. The one exception to this -is nodejs, which is installed from the `nodesource `_ -apt repository. The Ubuntu provided version of nodejs is usually too old. +via ``apt`` from the base Ubuntu repositories. We generally do not pin versions of packages provided by apt, instead just using the latest versions provided by Ubuntu. \ No newline at end of file diff --git a/docs/topic/installer-actions.rst b/docs/topic/installer-actions.rst index b57ae0e..caad82a 100644 --- a/docs/topic/installer-actions.rst +++ b/docs/topic/installer-actions.rst @@ -4,21 +4,19 @@ What does the installer do? =========================== -This document details what exactly the installer does to the machine it is +This document details what exactly the installer does to the machine it is run on. ``apt`` Packages installed ========================== The packages ``python3`` and ``python3-venv`` are installed from the apt repositories. -Since we need an recent & supported version of ``nodejs``, we install it from -`nodesource `_. Hub environment =============== JupyterHub is run from a python3 virtual environment located in ``/opt/tljh/hub``. It -uses the system's installed python and is owned by root. It also contains a binary install +uses the system's installed python and is owned by root. It also contains a binary install of `traefik `_. This virtual environment is completely managed by TLJH. .. note:: @@ -34,7 +32,7 @@ User environment By default, a ``miniconda`` environment is installed in ``/opt/tljh/user``. This contains the notebook interface used to launch all users, and the various packages available to all -users. The environment is owned by the ``root`` user. JupyterHub admins may use +users. The environment is owned by the ``root`` user. JupyterHub admins may use to ``sudo -E conda install`` or ``sudo -E pip install`` packages into this environment. This conda environment is added to ``$PATH`` for all users started with JupyterHub. If you @@ -49,7 +47,7 @@ This should let you run various ``conda`` and ``pip`` commands. If you run into .. code-block:: bash - sudo env PATH=${PATH} + sudo env PATH=${PATH} By default, ``sudo`` does not respect any custom environments you have activated. The ``env PATH=${PATH}`` 'fixes' that. diff --git a/tests/test_installer.py b/tests/test_installer.py index 7d594d6..ca13cc5 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -8,11 +8,6 @@ from tljh import installer from tljh.yaml import yaml -def test_ensure_node(): - installer.ensure_node() - assert os.path.exists('/usr/bin/node') - - def test_ensure_config_yaml(tljh_dir): pm = installer.setup_plugins() installer.ensure_config_yaml(pm) diff --git a/tljh/installer.py b/tljh/installer.py index 5852963..8c89ccf 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -42,67 +42,6 @@ HERE = os.path.abspath(os.path.dirname(__file__)) logger = logging.getLogger("tljh") -def ensure_node(): - """ - Ensure nodejs from nodesource is installed - """ - key = b""" ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 -Comment: GPGTools - https://gpgtools.org - -mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+ -W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs -fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy -qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7 -89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD -Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7 -C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/ -G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc -Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft -qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm -EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB -tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2 -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy -jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ -kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1 -GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm -XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU -VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka -1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI -IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc -pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn -xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y -gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI -AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ -fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ -Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh -41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea -JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD -xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi -vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/ -aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o -QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK -yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3 -QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek -fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK -CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec -0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1 -PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh -qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15 -ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac -hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb -DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4 -xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu -G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd -sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy -/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g== -=CLGF ------END PGP PUBLIC KEY BLOCK----- - """.strip() - apt.trust_gpg_key(key) - apt.add_source('nodesource', 'https://deb.nodesource.com/node_10.x', 'main') - apt.install_packages(['nodejs']) def remove_chp(): """ @@ -476,7 +415,6 @@ def main(): ensure_user_environment(args.user_requirements_txt_url) logger.info("Setting up JupyterHub...") - ensure_node() ensure_jupyterhub_package(HUB_ENV_PREFIX) # Stop the http server with the progress page before traefik starts From dc30923658cdf4ccbabe3a274bdb4204b5a37b3c Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Tue, 9 Feb 2021 09:07:51 +0100 Subject: [PATCH 4/5] Update to jupyter-resource-usage --- docs/howto/admin/nbresuse.rst | 4 ++-- docs/howto/admin/resize.rst | 22 +++++++++++----------- docs/howto/admin/resource-estimation.rst | 4 ++-- integration-tests/test_extensions.py | 4 ++-- tljh/requirements-base.txt | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/howto/admin/nbresuse.rst b/docs/howto/admin/nbresuse.rst index 585c7d5..9991406 100644 --- a/docs/howto/admin/nbresuse.rst +++ b/docs/howto/admin/nbresuse.rst @@ -4,7 +4,7 @@ Check your memory usage ======================= -The `nbresuse `_ extension is part of +The `jupyter-resource-usage `_ extension is part of the default installation, and tells you how much memory your user is using right now, and what the memory limit for your user is. It is shown in the top right corner of the notebook interface. Note that this is memory usage @@ -12,4 +12,4 @@ for everything your user is running through the Jupyter notebook interface, not just the specific notebook it is shown on. .. image:: ../../images/nbresuse.png - :alt: Memory limit / usage shown with nbresuse + :alt: Memory limit / usage shown with jupyter-resource-usage diff --git a/docs/howto/admin/resize.rst b/docs/howto/admin/resize.rst index 8cf1043..99e99c7 100644 --- a/docs/howto/admin/resize.rst +++ b/docs/howto/admin/resize.rst @@ -1,10 +1,10 @@ .. _howto/admin/resize: ================================================= -Resize the resources available to your JupyterHub +Resize the resources available to your JupyterHub ================================================= -As you are using your JupyterHub, you may need to increase or decrease +As you are using your JupyterHub, you may need to increase or decrease the amount of resources allocated to your TLJH install. The kinds of resources that can be allocated, as well as the process to do so, will depend on the provider / interface for your VM. We recommend consulting the installation page for your provider for more information. This @@ -18,12 +18,12 @@ Currently there are instructions to resize your resources on the following provi Once resources have been reallocated, you must tell TLJH to make use of these resources, and verify that the resources have become available. -Verifying a Resize +Verifying a Resize ================== -#. Once you have resized your server, tell the JupyterHub to make use of - these new resources. To accomplish this, follow the instructions in - :ref:`topic/tljh-config` to set new memory or CPU limits and reload the hub. This can be completed +#. Once you have resized your server, tell the JupyterHub to make use of + these new resources. To accomplish this, follow the instructions in + :ref:`topic/tljh-config` to set new memory or CPU limits and reload the hub. This can be completed using the terminal in the JupyterHub (or via SSH-ing into your VM and using this terminal). #. TLJH configuration options can be verified by viewing the tljh-config output. @@ -36,14 +36,14 @@ Verifying a Resize #. **To verify changes to memory**, confirm that it worked by starting a new server (if you had one previously running, click "Control Panel -> Stop My Server" to - shut down your active server first), opening a notebook, and checking the value of the - `nbresuse `_ extension in the upper-right. + shut down your active server first), opening a notebook, and checking the value of the + `jupyter-resource-usage `_ extension in the upper-right. .. image:: ../../images/nbresuse.png - :alt: nbresuse demonstration + :alt: jupyter-resource-usage demonstration -#. **To verify changes to CPU**, use the ``nproc`` from a terminal. - This command displays the number of available cores, and should be equal to the +#. **To verify changes to CPU**, use the ``nproc`` from a terminal. + This command displays the number of available cores, and should be equal to the number of cores you selected in your provider's interface. .. code-block:: bash diff --git a/docs/howto/admin/resource-estimation.rst b/docs/howto/admin/resource-estimation.rst index 3f00a92..db9d55b 100644 --- a/docs/howto/admin/resource-estimation.rst +++ b/docs/howto/admin/resource-estimation.rst @@ -62,10 +62,10 @@ stop, unlike with RAM. Recommended\, CPU = (Max\, concurrent\, users \times Max\, CPU\, usage\, per\, user) + 20\% -The ``20%`` is overhead for TLJH and related services. This is around 20% of a +The ``20%`` is overhead for TLJH and related services. This is around 20% of a single modern CPU. This, of course, is just an estimate. We recommend using the same process used to estimate Memory required for estimating CPU required. -You cannot use nbresuse for this, but you should carry out normal workflow and +You cannot use jupyter-resource-usage for this, but you should carry out normal workflow and investigate the CPU usage on the machine. Disk space diff --git a/integration-tests/test_extensions.py b/integration-tests/test_extensions.py index a56bd16..7c0809e 100644 --- a/integration-tests/test_extensions.py +++ b/integration-tests/test_extensions.py @@ -16,7 +16,7 @@ def test_serverextensions(): 'jupyterlab 3.', 'nbgitpuller 0.9.', 'nteract_on_jupyter 2.1.', - 'nbresuse ' + 'jupyter-resource-usage ' ] for e in extensions: @@ -33,7 +33,7 @@ def test_nbextensions(): ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) extensions = [ - 'nbresuse/main', + 'jupyter-resource-usage/main', # This is what ipywidgets nbextension is called 'jupyter-js-widgets/extension' ] diff --git a/tljh/requirements-base.txt b/tljh/requirements-base.txt index e4a6e97..4127fa4 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-base.txt @@ -10,8 +10,8 @@ nteract-on-jupyter==2.1.* # Install jupyterlab extensions from PyPI # nbgitpuller for easily pulling in Git repositories nbgitpuller==0.9.* -# nbresuse to show people how much RAM they are using -nbresuse==0.3.* +# jupyter-resource-usage to show people how much RAM they are using +jupyter-resource-usage==0.5.* # Most people consider ipywidgets to be part of the core notebook experience ipywidgets==7.6.* # Pin tornado From 1a4748bd0686776dae68bb8c6d9ac7b267b917ea Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Tue, 9 Feb 2021 10:34:12 +0100 Subject: [PATCH 5/5] Fix jupyter-resource-usage config --- integration-tests/test_extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/test_extensions.py b/integration-tests/test_extensions.py index 7c0809e..2d48c9b 100644 --- a/integration-tests/test_extensions.py +++ b/integration-tests/test_extensions.py @@ -16,7 +16,7 @@ def test_serverextensions(): 'jupyterlab 3.', 'nbgitpuller 0.9.', 'nteract_on_jupyter 2.1.', - 'jupyter-resource-usage ' + 'jupyter_resource_usage' ] for e in extensions: @@ -33,7 +33,7 @@ def test_nbextensions(): ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) extensions = [ - 'jupyter-resource-usage/main', + 'jupyter_resource_usage/main', # This is what ipywidgets nbextension is called 'jupyter-js-widgets/extension' ]