Update py-pyspark and py-py4j (#44263)

* Adding a py4j variant that requires Java via spack to avoid situations where a system doesn't have Java and py4j expects it
* Adding new versions of py-pyspark
* Adding a new variant to require java (via py4j) and clean up dependency handling
* Adding myself as a maintainer for py-pyspark and py-py4j
* Fix overlooked version bump in py4j
* Version bump to meet py-spark expectations
* Version bump to add latest compatibile version with pyspark
* Matching py-grpcio bump
* Adding variants and dependents for pyspark
* Adding runtime deps
* Changing default java requirement. I'm not sure this is the right call
* Changing py4j with java dependency handling
* Fix style
* Update package.py fix unnecessary f-strinh
* Make +java the default for both
* Fix nested deps
* Revert styles after default change
* Added new versions and GCC 14 compatbility conflicts
* Added new versions and compatibility conflicts for gcc 14
* Added new versions paired to arrow (for gcc14 compat)
* Update py-protobuf compiler conflict
* Update depends to match 
    See https://github.com/grpc/grpc/blob/master/src/python/grpcio_status/setup.py
* Updating dependencies and conflicts for py-googleapis-common-protos. Added new version to avoid future issues
* Remove upper bound version on py-protobuf and add default_args
* Adding new versions and updating dependencies back to versions 1.35.0
* Updating oldest numpy deps
* Fixing merge
* bit more cleaniness for var/spack/repos/builtin/packages/py-googleapis-common-protos/package.py
* Adding latest matching version of py-grpcio and py-grpcio-status
* Update package.py
   https://github.com/spack/spack/pull/44263#discussion_r1612317943
* Update dependencies
* Adding additional versions for dependent packages. Deprecated two versions: 1.16 is old, built for python ~3.6, and does not build for 3.8. 1.52.0 was removed from pypi
* Revert py-grpcio-tools changes. Will include in separate PR
* Adding patches and constraints to get 1.48 to build as it's a dependency that is called out for some other packages
* Updating to account for yanked packages for dependencies
* Fix style
* Update sha256 for py-grpcio v0.16.0 to reflect change

---------

Co-authored-by: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>

var/spack/repos/builtin/packages/arrow/package.py

@@ -17,6 +17,8 @@ class Arrow(CMakePackage, CudaPackage):
 
     license("Apache-2.0")
 
+    version("16.1.0", sha256="9762d9ecc13d09de2a03f9c625a74db0d645cb012de1e9a10dfed0b4ddc09524")
+    version("15.0.2", sha256="4735b349845bff1fe95ed11abbfed204eb092cabc37523aa13a80cb830fe5b5e")
     version("14.0.2", sha256="07cdb4da6795487c800526b2865c150ab7d80b8512a31793e6a7147c8ccd270f")
     version("14.0.1", sha256="a48e54a09d58168bc04d86b13e7dab04f0aaba18a6f7e4dadf3e9c7bb835c8f1")
     version("14.0.0", sha256="39e3388bbaba23faa7a5e8a82ebba7fe4c38ace2c394d6a3f26559715b30f401")
@@ -42,6 +44,7 @@ class Arrow(CMakePackage, CudaPackage):
     depends_on("boost@1.60: +filesystem +system")
     depends_on("cmake@3.2.0:", type="build")
     depends_on("flatbuffers")
+    conflicts("%gcc@14", when="@:15.0.1")  # https://github.com/apache/arrow/issues/40009
     depends_on("llvm@:11 +clang", when="+gandiva @:3", type="build")
     depends_on("llvm@:12 +clang", when="+gandiva @:4", type="build")
     depends_on("llvm@:13 +clang", when="+gandiva @:7", type="build")

var/spack/repos/builtin/packages/py-googleapis-common-protos/package.py

@@ -14,7 +14,9 @@ class PyGoogleapisCommonProtos(PythonPackage):
 
     license("Apache-2.0")
 
+    version("1.63.0", sha256="17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e")
     version("1.58.0", sha256="c727251ec025947d545184ba17e3578840fc3a24a0516a020479edab660457df")
+    version("1.56.4", sha256="c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417")
     version("1.55.0", sha256="53eb313064738f45d5ac634155ae208e121c963659627b90dfcb61ef514c03e1")
     version("1.6.0", sha256="e61b8ed5e36b976b487c6e7b15f31bb10c7a0ca7bd5c0e837f4afab64b53a0c6")
 
@@ -26,14 +28,21 @@ class PyGoogleapisCommonProtos(PythonPackage):
         description="Enable support for gRPC Remote Procedure Call framework.",
     )
 
-    with when("+grpc"):
-        depends_on("py-grpcio@1.44:1", when="@1.57:", type="run")
-        depends_on("py-grpcio@1", when="@1.56.2:", type="run")
-        depends_on("py-grpcio@1:", type="run")
-
     depends_on("py-setuptools", type="build")
-    depends_on(
+
-        "py-protobuf@3.19.5:3.19,3.20.2:4.21.0,4.21.6:4", when="@1.58:", type=("build", "run")
+    with default_args(type=("build", "run")):
-    )
+        # https://github.com/googleapis/python-api-common-protos/blob/main/setup.py
-    depends_on("py-protobuf@3.12.0:3", when="@1.55", type=("build", "run"))
+        # May be able to rais max version to :5 in next release
-    depends_on("py-protobuf@3.6.0:3", when="@:1.6", type=("build", "run"))
+        depends_on("py-protobuf@3.19.5:4", when="@1.58:")
+        depends_on("py-protobuf@3.15.0:4", when="@1.56:1.57")
+        depends_on("py-protobuf@3.12.0:4", when="@1.55")
+        depends_on("py-protobuf@3.6.0:", when="@1.6.0:")
+
+    # Explicitly incompatibile versions per setup.py
+    # https://github.com/googleapis/python-api-common-protos/issues/128
+    conflicts("py-protobuf@3.20:3.20.1")
+    conflicts("py-protobuf@4.21.1:4.21.5")
+
+    with when("+grpc"), default_args(type="run"):
+        depends_on("py-grpcio@1.44:1", when="@1.57:")
+        depends_on("py-grpcio@1:")

var/spack/repos/builtin/packages/py-grpcio-status/package.py

@@ -14,12 +14,18 @@ class PyGrpcioStatus(PythonPackage):
 
     license("Apache-2.0")
 
+    # Versions 1.63.0 and 1.64.0 are released but not yet on pypi
+
+    version("1.62.2", sha256="62e1bfcb02025a1cd73732a2d33672d3e9d0df4d21c12c51e0bbcaf09bab742a")
     version("1.60.1", sha256="61b5aab8989498e8aa142c20b88829ea5d90d18c18c853b9f9e6d407d37bf8b4")
+    version("1.56.2", sha256="a046b2c0118df4a5687f4585cca9d3c3bae5c498c4dff055dcb43fb06a1180c8")
 
     # https://github.com/grpc/grpc/blob/v1.60.1/src/python/grpcio_status/setup.py
 
-    depends_on("py-protobuf@4.21.6:", type=("build", "run"))
+    with default_args(type=("build", "run")):
-    depends_on("py-grpcio@1.60.1:", when="@1.60.1", type=("build", "run"))
+        depends_on("py-protobuf@4.21.6:")
-    depends_on("py-googleapis-common-protos@1.5.5:", type=("build", "run"))
+        for grpcio in ("1.62.2", "1.60.1", "1.56.2"):
+            depends_on(f"py-grpcio@{grpcio}", when=f"@{grpcio}")
+        depends_on("py-googleapis-common-protos@1.5.5:")
 
     depends_on("py-setuptools", type="build")

var/spack/repos/builtin/packages/py-grpcio-tools/package.py

@@ -12,6 +12,9 @@ class PyGrpcioTools(PythonPackage):
     homepage = "https://grpc.io/"
     pypi = "grpcio-tools/grpcio-tools-1.42.0.tar.gz"
 
+    version("1.62.2", sha256="5fd5e1582b678e6b941ee5f5809340be5e0724691df5299aae8226640f94e18f")
+    version("1.56.2", sha256="82af2f4040084141a732f0ef1ecf3f14fdf629923d74d850415e4d09a077e77a")
+    version("1.48.2", sha256="8902a035708555cddbd61b5467cea127484362decc52de03f061a1a520fe90cd")
     version("1.48.1", sha256="1178f2ea531f80cc2027ec64728df6ffc8e98cf1df61652a496eafd612127183")
     version("1.42.0", sha256="d0a0daa82eb2c2fb8e12b82a458d1b7c5516fe1135551da92b1a02e2cba93422")
     version("1.39.0", sha256="39dfe7415bc0d3860fdb8dd90607594b046b88b57dbe64284efa4820f951c805")
@@ -20,6 +23,9 @@ class PyGrpcioTools(PythonPackage):
     depends_on("py-setuptools", type="build")
     depends_on("py-protobuf@3.12.0:3", when="@1.48.1:", type=("build", "run"))
     depends_on("py-protobuf@3.5.0.post1:3", type=("build", "run"))
+    depends_on("py-grpcio@1.62.2:", when="@1.62.2:", type=("build", "run"))
+    depends_on("py-grpcio@1.56.2:", when="@1.56.2:", type=("build", "run"))
+    depends_on("py-grpcio@1.48.2:", when="@1.48.2:", type=("build", "run"))
     depends_on("py-grpcio@1.48.1:", when="@1.48.1:", type=("build", "run"))
     depends_on("py-grpcio@1.42.0:", when="@1.42.0:", type=("build", "run"))
     depends_on("py-grpcio@1.39.0:", when="@1.39.0:1.41", type=("build", "run"))

var/spack/repos/builtin/packages/py-grpcio/30522.diff

@@ -0,0 +1,21 @@
+diff --git a/setup.py b/setup.py
+index 8c428cb7f32b4..2646af5fd5ebb 100644
+--- a/setup.py
++++ b/setup.py
+@@ -214,11 +214,11 @@ def check_linker_need_libatomic():
+         return False
+     # Double-check to see if -latomic actually can solve the problem.
+     # https://github.com/grpc/grpc/issues/22491
+-    cpp_test = subprocess.Popen(
+-        [cxx, '-x', 'c++', '-std=c++14', '-', '-latomic'],
+-        stdin=PIPE,
+-        stdout=PIPE,
+-        stderr=PIPE)
++    cpp_test = subprocess.Popen(cxx +
++                                ['-x', 'c++', '-std=c++14', '-', '-latomic'],
++                                stdin=PIPE,
++                                stdout=PIPE,
++                                stderr=PIPE)
+     cpp_test.communicate(input=code_test)
+     return cpp_test.returncode == 0
+ 

var/spack/repos/builtin/packages/py-grpcio/package.py

@@ -14,9 +14,22 @@ class PyGrpcio(PythonPackage):
 
     license("Apache-2.0")
 
+    version("1.64.0", sha256="257baf07f53a571c215eebe9679c3058a313fd1d1f7c4eede5a8660108c52d9c")
+    version("1.63.0", sha256="f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1")
+    version("1.62.2", sha256="c77618071d96b7a8be2c10701a98537823b9c65ba256c0b9067e0594cdbd954d")
     version("1.60.1", sha256="dd1d3a8d1d2e50ad9b59e10aa7f07c7d1be2b367f3f2d33c5fade96ed5460962")
-    version("1.52.0", sha256="a5d4a83d29fc39af429c10b9b326c174fec49b73398e4a966a1f2a4f30aa4fdb")
+    version("1.56.2", sha256="0ff789ae7d8ddd76d2ac02e7d13bfef6fc4928ac01e1dcaa182be51b6bcc0aaa")
-    version("1.48.1", sha256="660217eccd2943bf23ea9a36e2a292024305aec04bf747fbcff1f5032b83610e")
+    version(
+        "1.52.0",
+        sha256="a5d4a83d29fc39af429c10b9b326c174fec49b73398e4a966a1f2a4f30aa4fdb",
+        deprecated=True,  # https://github.com/grpc/grpc/issues/32306
+    )
+    version("1.48.2", sha256="90e5da224c6b9b23658adf6f36de6f435ef7dbcc9c5c12330314d70d6f8de1f7")
+    version(
+        "1.48.1",
+        sha256="660217eccd2943bf23ea9a36e2a292024305aec04bf747fbcff1f5032b83610e",
+        deprecated=True,  # https://github.com/grpc/grpc/issues/30372
+    )
     version("1.43.0", sha256="735d9a437c262ab039d02defddcb9f8f545d7009ae61c0114e19dda3843febe5")
     version("1.42.0", sha256="4a8f2c7490fe3696e0cdd566e2f099fb91b51bc75446125175c55581c2f7bc11")
     version("1.39.0", sha256="57974361a459d6fe04c9ae0af1845974606612249f467bbd2062d963cb90f407")
@@ -36,16 +49,38 @@ class PyGrpcio(PythonPackage):
     version("1.28.1", sha256="cbc322c5d5615e67c2a15be631f64e6c2bab8c12505bc7c150948abdaa0bdbac")
     version("1.27.2", sha256="5ae532b93cf9ce5a2a549b74a2c35e3b690b171ece9358519b3039c7b84c887e")
     version("1.25.0", sha256="c948c034d8997526011960db54f512756fb0b4be1b81140a15b4ef094c6594a4")
-    version("1.16.0", sha256="d99db0b39b490d2469a8ef74197d5f211fa740fc9581dccecbb76c56d080fce1")
+    version(
+        "1.16.0",
+        sha256="0cc5f2d3ee21c642d8982f197c83053fd3a8cbcd6a60240d8c87c6c256b10d57",
+        deprecated=True,
+        # Released 6ish years ago and does not install for python 3.8 with gcc11
+    )
 
     depends_on("py-setuptools", type="build")
-    depends_on("py-six@1.5.2:", when="@:1.48", type=("build", "run"))
+    with default_args(type=("build", "run")):
-    depends_on("py-cython@0.23:2", type="build")
+        depends_on("py-coverage@4:")
+        depends_on("py-wheel@0.29:")
+
+        depends_on("py-cython@3:", when="@1.63.0:")
+        depends_on("py-cython@0.29.8:2", when="@1.56.0:1.62")
+        depends_on("py-cython@0.29.8:", when="@1.49.0:1.55")
+        # States dependency in setup.py >=0.23
+        # Package states >=0.23 but doesn't compile w/ >=3
+        depends_on("py-cython@0.23:2", when="@:1.48")
+
+        depends_on("py-protobuf@5.26.1:5", when="@1.63.0:")
+        depends_on("py-protobuf@4.21.3:4", when="@1.49.0:1.62")
+        depends_on("py-protobuf@3.5.0:3", when="@:1.48")
+
+        depends_on("py-six@1.10:", when="@:1.48")
+
     depends_on("openssl")
     depends_on("zlib-api")
     depends_on("c-ares")
     depends_on("re2+shared")
 
+    patch("30522.diff", when="@1.48")  # https://github.com/grpc/grpc/issues/30372
+
     def setup_build_environment(self, env):
         env.set("GRPC_PYTHON_BUILD_WITH_CYTHON", True)
         env.set("GRPC_PYTHON_BUILD_SYSTEM_OPENSSL", True)

var/spack/repos/builtin/packages/py-protobuf/package.py

@@ -17,6 +17,8 @@ class PyProtobuf(PythonPackage):
     homepage = "https://developers.google.com/protocol-buffers/"
     pypi = "protobuf/protobuf-3.11.0.tar.gz"
 
+    version("5.26.1", sha256="8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51")
+    version("4.25.3", sha256="25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c")
     version("4.24.3", sha256="12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d")
     version("4.23.3", sha256="7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b")
     version("4.21.9", sha256="61f21493d96d2a77f9ca84fefa105872550ab5ef71d21c458eb80edcf4885a99")
@@ -76,6 +78,7 @@ class PyProtobuf(PythonPackage):
         depends_on(f"protobuf@3.{ver}", when=f"@3.{ver}+cpp")
 
     conflicts("+cpp", when="^python@3.11:")
+    conflicts("%gcc@14", when="@:4.24.3")
 
     @property
     def build_directory(self):

var/spack/repos/builtin/packages/py-py4j/package.py

@@ -15,6 +15,9 @@ class PyPy4j(PythonPackage):
 
     license("BSD-3-Clause")
 
+    maintainers("teaguesterling")
+
+    version("0.10.9.7", sha256="0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb")
     version("0.10.9.5", sha256="276a4a3c5a2154df1860ef3303a927460e02e97b047dc0a47c1c3fb8cce34db6")
     version("0.10.9.3", sha256="0d92844da4cb747155b9563c44fc322c9a1562b3ef0979ae692dbde732d784dd")
     version("0.10.9", sha256="36ec57f43ff8ced260a18aa9a4e46c3500a730cac8860e259cbaa546c2b9db2f")
@@ -23,7 +26,10 @@ class PyPy4j(PythonPackage):
     version("0.10.4", sha256="406fbfdbcbbb398739f61fafd25724670a405a668eb08c1721d832eadce06aae")
     version("0.10.3", sha256="f4570108ad014dd52a65c2288418e31cb8227b5ecc39ad7fc7fe98314f7a26f2")
 
+    variant("java", default=True, description="Require java via Spack instead of using system JRE")
+
     depends_on("py-setuptools", type="build")
+    depends_on("java", when="+java", type="run")
 
     def url_for_version(self, version):
         url = "https://pypi.io/packages/source/p/py4j/"

var/spack/repos/builtin/packages/py-pyarrow/package.py

@@ -20,6 +20,8 @@ class PyPyarrow(PythonPackage, CudaPackage):
 
     license("Apache-2.0")
 
+    version("16.1.0", sha256="15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315")
+    version("15.0.2", sha256="9c9bc803cb3b7bfacc1e96ffbfd923601065d9d3f911179d81e72d99fd74a3d9")
     version("14.0.2", sha256="36cef6ba12b499d864d1def3e990f97949e0b79400d08b7cf74504ffbd3eb025")
     version("13.0.0", sha256="83333726e83ed44b0ac94d8d7a21bbdee4a05029c3b1e8db58a863eec8fd8a33")
     version("12.0.1", sha256="cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec")
@@ -61,7 +63,9 @@ class PyPyarrow(PythonPackage, CudaPackage):
     depends_on("py-pip@:23.0", type="build")
 
     depends_on("py-numpy@1.16.6:", type=("build", "run"), when="@3:")
-    depends_on("py-numpy@1.14:", type=("build", "run"), when="@0.15:")
+    # Prior to python 3.9 numpy must be >=0.14,<1.25
+    depends_on("py-numpy@0.14:1.24", when="^python@:3.8")
+    depends_on("py-numpy@1.25:", when="^python@3.9:")
 
     arrow_versions = (
         "@0.9.0",
@@ -78,6 +82,8 @@ class PyPyarrow(PythonPackage, CudaPackage):
         "@12.0.1",
         "@13.0.0",
         "@14.0.2",
+        "@15.0.2",
+        "@16.1.0",
     )
     for v in arrow_versions:
         depends_on("arrow+python" + v, when=v)

var/spack/repos/builtin/packages/py-pyspark/package.py

@@ -12,6 +12,11 @@ class PyPyspark(PythonPackage):
     homepage = "https://spark.apache.org"
     pypi = "pyspark/pyspark-3.0.1.tar.gz"
 
+    maintainers("teaguesterling")
+
+    version("3.5.1", sha256="dd6569e547365eadc4f887bf57f153e4d582a68c4b490de475d55b9981664910")
+    version("3.4.3", sha256="8d7025fa274830cb6c3bd592228be3d9345cb3b8b1e324018c2aa6e75f48a208")
+    version("3.3.4", sha256="1f866be47130a522355240949ed50d9812a8f327bd7619f043ffe07fbcf7f7b6")
     version("3.3.1", sha256="e99fa7de92be406884bfd831c32b9306a3a99de44cfc39a2eefb6ed07445d5fa")
     version("3.3.0", sha256="7ebe8e9505647b4d124d5a82fca60dfd3891021cf8ad6c5ec88777eeece92cf7")
     version("3.2.1", sha256="0b81359262ec6e9ac78c353344e7de026027d140c6def949ff0d80ab70f89a54")
@@ -19,11 +24,40 @@ class PyPyspark(PythonPackage):
     version("3.1.2", sha256="5e25ebb18756e9715f4d26848cc7e558035025da74b4fc325a0ebc05ff538e65")
     version("3.0.1", sha256="38b485d3634a86c9a2923c39c8f08f003fdd0e0a3d7f07114b2fb4392ce60479")
 
+    variant("java", default=True, description="Include Java requirements via py-py4j")
+    variant("pandas", default=True, description="Include Pandas support")
+    variant("connect", default=True, description="Include SparkConnect support", when="@3.4:")
+
+    # Noted on https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies
+    with default_args(type="run"):
+        depends_on("py-pyarrow@4:", when="+pandas@3.5:")
+        depends_on("py-pyarrow@1:", when="+pandas@:3.4")
+        depends_on("py-pandas@1.0.5:", when="+pandas")
+        depends_on("py-numpy@1.15:", when="+pandas")
+
+        with when("@3.5:"):
+            depends_on("py-grpcio@1.56:", when="+connect")
+            depends_on("py-grpcio-status@1.56:", when="+connect")
+            depends_on("py-googleapis-common-protos@1.56.4:", when="+connect")
+
+        with when("@3.4:"):
+            depends_on("py-grpcio@1.48.1:", when="+connect")
+            depends_on("py-grpcio-status@1.48.1:", when="+connect")
+            depends_on("py-googleapis-common-protos@1.56.4:", when="+connect")
+
     depends_on("py-setuptools", type="build")
-    depends_on("py-py4j@0.10.9.5", when="@3.3.0:", type=("build", "run"))
+    with default_args(type=("build", "run")):
-    depends_on("py-py4j@0.10.9.3", when="@3.2.1", type=("build", "run"))
+        depends_on("py-py4j~java", when="~java")
-    depends_on("py-py4j@0.10.9", when="@3.0.1:3.1.3", type=("build", "run"))
+        for py4j_version, pyspark_version in [
+            ("0.10.9.7", "3.4:"),
+            ("0.10.9.5", "3.3:"),
+            ("0.10.9.3", "3.2.1"),
+            ("0.10.9", "3.0.1:3.1.3"),
+        ]:
+            depends_on(f"py-py4j@{py4j_version}:", when=f"@{pyspark_version}")
 
     def setup_run_environment(self, env):
         env.set("PYSPARK_PYTHON", python.path)
         env.set("PYSPARK_DRIVER_PYTHON", python.path)
+        if self.spec.satisfies("+pandas ^java@11:"):
+            env.append_flags("SPARK_SUBMIT_OPTS", "-Dio.netty.tryReflectionSetAccessible=true")