Migrate as much as possible to pyproject.toml (#12850)

I migrated as much build logic as possible to pyproject.toml for cudf, dask_cudf, and custreamz. A couple of notes: - I didn't move cudf_kafka because it is currently still using a pure setuptools build with Cython files. We may as well do the pyproject.toml migration there after we have also done the scikit-build conversion since there's a lot of logic that can't be removed from setup.py until then. I'm also not entirely confident that everything will work as expected if we move data to pyproject.toml without switching to scikit-build based on issues like the ones mentioned in [this SO post about Cython and project data stored in pyproject.toml](https://stackoverflow.com/questions/73800736/pyproject-toml-and-cython-extension-module). - I could get rid of setup.py entirely for custreamz at this point, but I expect that we'll run into a host of additional issues when we do eventually move to requiring `pip install X` instead of `python setup.py X` across RAPIDS and I'd rather deal with all of those at the same time with our main packages rather than using an esoteric package like custreamz as the test case. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Ray Douglass (https://github.com/raydouglass) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: #12850
rapidsai · Mar 6, 2023 · 7dade51 · 7dade51
1 parent 39398e4
commit 7dade51
Show file tree

Hide file tree

Showing 12 changed files with 198 additions and 153 deletions.
diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh
@@ -12,22 +12,19 @@ sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/dask_cudf/dask
 sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/cudf_kafka/cudf_kafka/__init__.py
 sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/custreamz/custreamz/__init__.py
 
-# setup.py versions
-sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf/setup.py
-sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/dask_cudf/setup.py
-sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf_kafka/setup.py
-sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/custreamz/setup.py
-
-# cudf setup.py cuda suffixes
-sed -i "s/name=\"cudf\"/name=\"cudf${CUDA_SUFFIX}\"/g" python/cudf/setup.py
-sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/setup.py
-sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/setup.py
-sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/setup.py
+# pyproject.toml versions
+sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf/pyproject.toml
+sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/dask_cudf/pyproject.toml
+sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf_kafka/pyproject.toml
+sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/custreamz/pyproject.toml
 
 # cudf pyproject.toml cuda suffixes
+sed -i "s/^name = \"cudf\"/name = \"cudf${CUDA_SUFFIX}\"/g" python/cudf/pyproject.toml
 sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
+sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
+sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
 
-# dask_cudf setup.py cuda suffixes
-sed -i "s/name=\"dask-cudf\"/name=\"dask-cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/setup.py
+# dask_cudf pyproject.toml cuda suffixes
+sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/pyproject.toml
 # Need to provide the == to avoid modifying the URL
-sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/setup.py
+sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -49,11 +49,11 @@ sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/dask
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cudf_kafka/cudf_kafka/__init__.py
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/custreamz/custreamz/__init__.py
 
-# Python setup.py updates
-sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf/setup.py
-sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/dask_cudf/setup.py
-sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf_kafka/setup.py
-sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/custreamz/setup.py
+# Python pyproject.toml updates
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cudf/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/dask_cudf/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cudf_kafka/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/custreamz/pyproject.toml
 
 # rapids-cmake version
 sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
@@ -89,12 +89,9 @@ sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_
 # Need to distutils-normalize the original version
 NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
 
-# Dependency versions in setup.py
-sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/setup.py
-sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/setup.py
-
 # Dependency versions in pyproject.toml
 sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/pyproject.toml
+sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/pyproject.toml
 
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"

diff --git a/python/cudf/MANIFEST.in b/python/cudf/MANIFEST.in
@@ -0,0 +1,16 @@
+# Cython files
+recursive-include cudf *.pxd
+recursive-include cudf *.pyx
+
+# Typing files
+recursive-include cudf *.pyi
+
+# C++ files
+recursive-include cudf *.hpp
+recursive-include udf_cpp *.hpp
+recursive-include udf_cpp *.cuh
+
+# Build files. Don't use a recursive include on '.' in case the repo is dirty
+include . CMakeLists.txt
+recursive-include cudf CMakeLists.txt
+recursive-include cmake *
diff --git a/python/cudf/README.md b/python/cudf/README.md
@@ -0,0 +1 @@
+../../README.md
diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml
@@ -16,6 +16,66 @@ requires = [
     "rmm==23.4.*",
 ]
 
+[project]
+name = "cudf"
+version = "23.04.00"
+description = "cuDF - GPU Dataframe"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "NVIDIA Corporation" },
+]
+license = { text = "Apache 2.0" }
+requires-python = ">=3.8"
+dependencies = [
+    "cachetools",
+    "cuda-python>=11.7.1,<12.0",
+    "fsspec>=0.6.0",
+    "numba>=0.56.2",
+    "numpy",
+    "nvtx>=0.2.1",
+    "packaging",
+    "pandas>=1.0,<1.6.0dev0",
+    "protobuf==4.21",
+    "typing_extensions",
+    # Allow floating minor versions for Arrow.
+    "pyarrow==10",
+    "rmm==23.4.*",
+    "ptxcompiler",
+    "cubinlinker",
+    "cupy-cuda11x",
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.10",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-benchmark",
+    "pytest-xdist",
+    "hypothesis",
+    "mimesis>=4.1.0",
+    "fastavro>=0.22.9",
+    "python-snappy>=0.6.0",
+    "pyorc",
+    "msgpack",
+    "transformers==4.24.0",
+    "tzdata",
+]
+
+[project.urls]
+Homepage = "https://github.com/rapidsai/cudf"
+Documentation = "https://docs.rapids.ai/api/cudf/stable/"
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+
 [tool.isort]
 line_length = 79
 multi_line_output = 3

diff --git a/python/cudf/setup.py b/python/cudf/setup.py
@@ -3,64 +3,8 @@
 from setuptools import find_packages
 from skbuild import setup
 
-install_requires = [
-    "cachetools",
-    "cuda-python>=11.7.1,<12.0",
-    "fsspec>=0.6.0",
-    "numba>=0.56.2",
-    "numpy",
-    "nvtx>=0.2.1",
-    "packaging",
-    "pandas>=1.0,<1.6.0dev0",
-    "protobuf==4.21",
-    "typing_extensions",
-    # Allow floating minor versions for Arrow.
-    "pyarrow==10",
-    "rmm==23.4.*",
-    "ptxcompiler",
-    "cubinlinker",
-    "cupy-cuda11x",
-]
-
-extras_require = {
-    "test": [
-        "pytest",
-        "pytest-benchmark",
-        "pytest-xdist",
-        "hypothesis",
-        "mimesis>=4.1.0",
-        "fastavro>=0.22.9",
-        "python-snappy>=0.6.0",
-        "pyorc",
-        "msgpack",
-        "transformers==4.24.0",
-        "tzdata",
-    ]
-}
-
 setup(
-    name="cudf",
-    version="23.04.00",
-    description="cuDF - GPU Dataframe",
-    url="https://github.com/rapidsai/cudf",
-    author="NVIDIA Corporation",
-    license="Apache 2.0",
-    classifiers=[
-        "Intended Audience :: Developers",
-        "Topic :: Database",
-        "Topic :: Scientific/Engineering",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
     include_package_data=True,
     packages=find_packages(include=["cudf", "cudf.*"]),
-    package_data={
-        key: ["*.pxd"] for key in find_packages(include=["cudf._lib*"])
-    },
-    install_requires=install_requires,
-    extras_require=extras_require,
     zip_safe=False,
 )
diff --git a/python/custreamz/LICENSE b/python/custreamz/LICENSE
@@ -0,0 +1 @@
+../../LICENSE
diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml
@@ -1,12 +1,57 @@
 # Copyright (c) 2021-2022, NVIDIA CORPORATION.
 
 [build-system]
-
+build-backend = "setuptools.build_meta"
 requires = [
     "wheel",
     "setuptools",
 ]
 
+[project]
+name = "custreamz"
+version = "23.04.00"
+description = "cuStreamz - GPU Accelerated Streaming"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "NVIDIA Corporation" },
+]
+license = { text = "Apache 2.0" }
+requires-python = ">=3.8"
+dependencies = [
+    "cudf",
+    "cudf_kafka",
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Topic :: Streaming",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Apache Kafka",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-xdist",
+]
+
+[project.urls]
+Homepage = "https://github.com/rapidsai/cudf"
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+zip-safe = false
+
+[tools.setuptools.packages.find]
+include = [
+    "custreamz",
+    "custreamz.*",
+]
+
 [tool.isort]
 line_length = 79
 multi_line_output = 3

diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py
@@ -1,31 +1,5 @@
 # Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
-from setuptools import find_packages, setup
+from setuptools import setup
 
-install_requires = ["cudf_kafka", "cudf"]
-
-extras_require = {"test": ["pytest", "pytest-xdist"]}
-
-setup(
-    name="custreamz",
-    version="23.04.00",
-    description="cuStreamz - GPU Accelerated Streaming",
-    url="https://github.com/rapidsai/cudf",
-    author="NVIDIA Corporation",
-    license="Apache 2.0",
-    classifiers=[
-        "Intended Audience :: Developers",
-        "Topic :: Streaming",
-        "Topic :: Scientific/Engineering",
-        "Topic :: Apache Kafka",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
-    packages=find_packages(include=["custreamz", "custreamz.*"]),
-    install_requires=install_requires,
-    extras_require=extras_require,
-    zip_safe=False,
-)
+setup()
diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml
@@ -1,12 +1,58 @@
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.
+# Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
 [build-system]
-
+build-backend = "setuptools.build_meta"
 requires = [
     "wheel",
     "setuptools",
 ]
 
+[project]
+name = "dask_cudf"
+version = "23.04.00"
+description = "Utilities for Dask and cuDF interactions"
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [
+    { name = "NVIDIA Corporation" },
+]
+license = { text = "Apache 2.0" }
+requires-python = ">=3.8"
+dependencies = [
+    "dask>=2023.1.1",
+    "distributed>=2023.1.1",
+    "fsspec>=0.6.0",
+    "numpy",
+    "pandas>=1.0,<1.6.0dev0",
+    "cudf==23.4.*",
+    "cupy-cuda11x",
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+]
+dynamic = ["entry-points"]
+
+[project.optional-dependencies]
+test = [
+    "numpy",
+    "pandas>=1.0,<1.6.0dev0",
+    "pytest",
+    "pytest-xdist",
+    "numba>=0.56.2",
+]
+
+[project.urls]
+Homepage = "https://github.com/rapidsai/cudf"
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+
 [tool.isort]
 line_length = 79
 multi_line_output = 3

diff --git a/python/dask_cudf/setup.cfg b/python/dask_cudf/setup.cfg