py-fairscale and dependencies py-pgzip libgit2 py-pygit2 (#39220)

* simple build of py-openai

* added variants to py-openai

* py-pandas-stubs is a dependency for py-openai

* fixed format and flake8 errors for py-openai

* black format error for py-pandas-stubs

* [@spackbot] updating style on behalf of sidpbury

* made style and format changes to py-openai

* made style and format changes to py-pandas-stubs

* py-types-pytz is a dependency for py-openai

* [@spackbot] updating style on behalf of sidpbury

* updated py-openpyxl for ver 3.0.7 and 3.1.2

* Update var/spack/repos/builtin/packages/py-pandas-stubs/package.py

Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>

* ajs requested changes for py-openai

* updated py-openpyxl for supported python

* [@spackbot] updating style on behalf of sidpbury

* updated py-openpyxl

* removed requirement.txt dependencies in  py-openpyxl

* removed python depends on from openpyxl

* updated package to support newer versions

* updated version of py-pygit2

* py-fairscale is a new package in support of torch

* py-pgzip is a dependency for py-fairscale

* switch fairscale pypi, added extra variant for convenience

* removed python dependency

* changed multiple requirement versions

* changes for upstream py-fairscale

* changes for upsteam py-pygit2

* Update package.py

* Update package.py

* sorted out some of the dependency versions

* removed version 1.12.2 because dependency could not be met

* updated py-cached-property dependency

* suggested changes from adamjstewart

---------

Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>

var/spack/repos/builtin/packages/libgit2/package.py

@@ -16,7 +16,13 @@ class Libgit2(CMakePackage):
     homepage = "https://libgit2.github.com/"
     url = "https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz"
 
+    version("1.7.0", sha256="d9d0f84a86bf98b73e68997f5c1543cc5067d0ca9c7a5acaba3e8d117ecefef3")
     version("1.6.4", sha256="d25866a4ee275a64f65be2d9a663680a5cf1ed87b7ee4c534997562c828e500d")
+    version("1.6.3", sha256="a8e2a09835eabb24ace2fd597a78af182e1e199a894e99a90e4c87c849fcd9c4")
+    version("1.6.2", sha256="d557fbf35557bb5df53cbf38ae0081edb4a36494ec0d19741fa673e509245f8f")
+    version("1.6.1", sha256="25f7d3e2e7f398477a6e18271c5a68510853d0fe826a6287811cebccf92e773f")
+    version("1.5.2", sha256="57638ac0e319078f56a7e17570be754515e5b1276d3750904b4214c92e8fa196")
+    version("1.5.1", sha256="7074f1e2697992b82402501182db254fe62d64877b12f6e4c64656516f4cde88")
     version("1.5.0", sha256="8de872a0f201b33d9522b817c92e14edb4efad18dae95cf156cf240b2efff93e")
     version("1.4.4", sha256="e9923e9916a32f54c661d55d79c28fa304cb23617639e68bff9f94d3e18f2d4b")
     version("1.4.3", sha256="f48b961e463a9e4e7e7e58b21a0fb5a9b2a1d24d9ba4d15870a0c9b8ad965163")

var/spack/repos/builtin/packages/py-fairscale/package.py

@@ -0,0 +1,37 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyFairscale(PythonPackage):
+    """FairScale is a PyTorch extension library for high performance and large
+    scale training. This library extends basic PyTorch capabilities while adding
+    new SOTA scaling techniques."""
+
+    homepage = "https://github.com/facebookresearch/fairscale"
+    pypi = "fairscale/fairscale-0.4.13.tar.gz"
+
+    version("0.4.13", sha256="1b797825c427f5dba92253fd0d8daa574e8bd651a2423497775fab1b30cfb768")
+    version("0.4.4", sha256="7719898743dc58c04a2294c896ee6308c92ccb3af9e10632b2a62f77cb689357")
+
+    variant("extra", default=False, description="support for cuda.list.gpu, scaler, weight")
+
+    # from setup.py
+    depends_on("python@3.8:", type=("build", "run"))
+    depends_on("ninja", type="build")
+    # from pyproject.toml
+    depends_on("py-wheel@0.30.0:", type="build")
+    depends_on("py-setuptools@40.6.2:", type="build")
+    # from requirements.txt
+    depends_on("py-torch@1.8.0:", type=("build", "run"))
+    depends_on("py-numpy@1.22.0:", type=("build", "run"))
+    # added extra to support cuda.list.gpu, scaler, and weight (not in pip install)
+    # see requirements-dev.txt in github. This does not include the tools for testing
+    with when("+extra"):
+        depends_on("py-pynvml@8.0.4", type=("build", "run"))
+        depends_on("py-scikit-learn@1.1.3", type=("build", "run"))
+        depends_on("py-pygit2@1.11.1", type=("build", "run"))
+        depends_on("py-pgzip@0.3.1", type=("build", "run"))

var/spack/repos/builtin/packages/py-pgzip/package.py

@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPgzip(PythonPackage):
+    """A multi-threading implementation of Python gzip module"""
+
+    homepage = "https://github.com/pgzip/pgzip"
+    pypi = "pgzip/pgzip-0.3.4.tar.gz"
+
+    version("0.3.4", sha256="ef56449039bc6e88558e46fe6bb11e3faaeef445d3985a9fb286795ff842c480")
+    version("0.3.1", sha256="a9c2df369311473ec3c239f26bf01638bdc6b6094d89ff4c81c6ef5c84eb24b7")
+
+    depends_on("py-setuptools", type="build")

var/spack/repos/builtin/packages/py-pygit2/package.py

@@ -12,30 +12,36 @@ class PyPygit2(PythonPackage):
     """
 
     homepage = "https://www.pygit2.org/"
-    pypi = "pygit2/pygit2-0.24.1.tar.gz"
+    pypi = "pygit2/pygit2-1.12.2.tar.gz"
 
+    version("1.12.1", sha256="56e85d0e66de957d599d1efb2409d39afeefd8f01009bfda0796b42a4b678358")
+    version("1.11.1", sha256="793f583fd33620f0ac38376db0f57768ef2922b89b459e75b1ac440377eb64ec")
     version("1.6.0", sha256="7aacea4e57011777f4774421228e5d0ddb9a6ddb87ac4b542346d17ab12a4d62")
     version("1.4.0", sha256="cbeb38ab1df9b5d8896548a11e63aae8a064763ab5f1eabe4475e6b8a78ee1c8")
     version("1.3.0", sha256="0be93f6a8d7cbf0cc79ae2f0afb1993fc055fc0018c27e2bd01ba143e51d4452")
     version("0.28.2", sha256="4d8c3fbbf2e5793a9984681a94e6ac2f1bc91a92cbac762dbdfbea296b917f86")
-    version("0.24.1", sha256="4d1d0196b38d6012faf0a7c45e235c208315672b6035da504566c605ba494064")
 
     depends_on("py-setuptools", type="build")
-    # Version must match with libgit2
+    # https://www.pygit2.org/install.html#version-numbers
-    # See: https://www.pygit2.org/install.html
+    depends_on("libgit2@1.6", when="@1.12")
-    depends_on("libgit2@1.1.0:1.1", when="@1.4:")
+    depends_on("libgit2@1.5", when="@1.10:1.11")
-    depends_on("libgit2@1.0.0:1.0", when="@1.2:1.3")
+    depends_on("libgit2@1.4", when="@1.9")
-    depends_on("libgit2@0:1.0", when="@1.1.0:1.1")
+    depends_on("libgit2@1.3", when="@1.7:1.8")
-    depends_on("libgit2@0.28.0:0.28", when="@0.28:1.0")
+    depends_on("libgit2@1.1", when="@1.4:1.6")
-    depends_on("libgit2@0.24:0.27", when="@0.24:0.27")
+    depends_on("libgit2@1.0", when="@1.2:1.3")
-    depends_on("python@3.6:", when="@1.4.0:")
+    depends_on("libgit2@0.99:1.0", when="@1.1")
-    depends_on("python@3.6:3.8", when="@1.2:1.3")
+    depends_on("libgit2@0.28", when="@0.28:1.0")
-    depends_on("python@3.5:3.8", when="@1.0:1.1")
+    depends_on("python@3.8:3.11", when="@1.11:1.12.1")
-    depends_on("python@2.7:3.7", when="@0.28:0")
+    depends_on("python@:3.10", when="@1.7:1.10")
+    depends_on("python@:3.9", when="@1.4:1.6")
+    depends_on("python@:3.8", when="@1.0:1.3")
+    depends_on("python@:3.7", when="@0.28")
     depends_on("py-six", type=("build", "run"), when="@:0.28.2")
-    depends_on("py-cffi@1.4.0:", type=("build", "run"))
+    depends_on("py-cffi@1.4.0:", when="@:1.5", type=("build", "run"))
-    depends_on("py-cached-property", when="@1.1.0:1.5", type=("build", "run"))
+    depends_on("py-cffi@1.6.0:", when="@1.6:1.7", type=("build", "run"))
-    depends_on("py-cached-property", when="@1.6.0: ^python@:3.7", type=("build", "run"))
+    depends_on("py-cffi@1.9.1:", when="@1.8:", type=("build", "run"))
+    depends_on("py-cached-property", when="@1.1:1.5", type=("build", "run"))
+    depends_on("py-cached-property", when="@1.6: ^python@:3.7", type=("build", "run"))
 
     def setup_build_environment(self, env):
         spec = self.spec