Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Migrate as much as possible to pyproject.toml #12850

Merged
merged 18 commits into from
Mar 6, 2023
Merged

Migrate as much as possible to pyproject.toml #12850

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
0a65a49
Migrate cudf.
vyasr Feb 24, 2023
f0b233a
Migrate custreamz.
vyasr Feb 24, 2023
32a859a
Migrate dask_cudf.
vyasr Feb 24, 2023
c685734
Move entry point back to setup.py as per scikit-build requirements.
vyasr Feb 24, 2023
3270a1e
Move zip-safe to setup.py.
vyasr Feb 24, 2023
8f827e8
Move remaining logic for streamz since it doesn't use scikit-build.
vyasr Feb 24, 2023
84d698b
Use MANIFEST for cudf.
vyasr Feb 24, 2023
91e10ce
Remove unnecessary README files.
vyasr Feb 24, 2023
4d8a2d7
Fix typo.
vyasr Feb 24, 2023
981e6eb
Update update-version.sh.
vyasr Feb 24, 2023
ff00d1b
Update apply_wheel_modifications.sh.
vyasr Feb 25, 2023
9a6d04c
Fix typo.
vyasr Feb 25, 2023
3332593
Fix regex for dask-cudf.
vyasr Feb 25, 2023
3c63fc0
Merge branch 'branch-23.04' into feat/all_pyproject
vyasr Mar 1, 2023
939ecda
Set entry points as dynamic.
vyasr Mar 2, 2023
17b17da
Fix typo.
vyasr Mar 2, 2023
c380951
Merge branch 'branch-23.04' into feat/all_pyproject
galipremsagar Mar 3, 2023
503d4a4
Merge branch 'branch-23.04' into feat/all_pyproject
vyasr Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions ci/release/apply_wheel_modifications.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,19 @@ sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/dask_cudf/dask
sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/cudf_kafka/cudf_kafka/__init__.py
sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/custreamz/custreamz/__init__.py

# setup.py versions
sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf/setup.py
sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/dask_cudf/setup.py
sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf_kafka/setup.py
sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/custreamz/setup.py

# cudf setup.py cuda suffixes
sed -i "s/name=\"cudf\"/name=\"cudf${CUDA_SUFFIX}\"/g" python/cudf/setup.py
sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/setup.py
sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/setup.py
sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/setup.py
# pyproject.toml versions
sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf/pyproject.toml
sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/dask_cudf/pyproject.toml
sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf_kafka/pyproject.toml
sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/custreamz/pyproject.toml

# cudf pyproject.toml cuda suffixes
sed -i "s/^name = \"cudf\"/name = \"cudf${CUDA_SUFFIX}\"/g" python/cudf/pyproject.toml
sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/pyproject.toml
sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/pyproject.toml

# dask_cudf setup.py cuda suffixes
sed -i "s/name=\"dask-cudf\"/name=\"dask-cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/setup.py
# dask_cudf pyproject.toml cuda suffixes
sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/pyproject.toml
# Need to provide the == to avoid modifying the URL
sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/setup.py
sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml
15 changes: 6 additions & 9 deletions ci/release/update-version.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/dask
sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cudf_kafka/cudf_kafka/__init__.py
sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/custreamz/custreamz/__init__.py

# Python setup.py updates
sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf/setup.py
sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/dask_cudf/setup.py
sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf_kafka/setup.py
sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/custreamz/setup.py
# Python pyproject.toml updates
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cudf/pyproject.toml
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/dask_cudf/pyproject.toml
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cudf_kafka/pyproject.toml
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/custreamz/pyproject.toml

# rapids-cmake version
sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
Expand Down Expand Up @@ -89,12 +89,9 @@ sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_
# Need to distutils-normalize the original version
NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")

# Dependency versions in setup.py
sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/setup.py
sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/setup.py

# Dependency versions in pyproject.toml
sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/pyproject.toml
sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/pyproject.toml

for FILE in .github/workflows/*.yaml; do
sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
Expand Down
16 changes: 16 additions & 0 deletions python/cudf/MANIFEST.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Cython files
recursive-include cudf *.pxd
recursive-include cudf *.pyx

# Typing files
recursive-include cudf *.pyi

# C++ files
recursive-include cudf *.hpp
recursive-include udf_cpp *.hpp
recursive-include udf_cpp *.cuh

# Build files. Don't use a recursive include on '.' in case the repo is dirty
include . CMakeLists.txt
recursive-include cudf CMakeLists.txt
recursive-include cmake *
1 change: 1 addition & 0 deletions python/cudf/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../README.md
60 changes: 60 additions & 0 deletions python/cudf/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,66 @@ requires = [
"rmm==23.4.*",
]

[project]
name = "cudf"
version = "23.04.00"
description = "cuDF - GPU Dataframe"
readme = { file = "README.md", content-type = "text/markdown" }
authors = [
{ name = "NVIDIA Corporation" },
]
license = { text = "Apache 2.0" }
requires-python = ">=3.8"
dependencies = [
"cachetools",
"cuda-python>=11.7.1,<12.0",
"fsspec>=0.6.0",
"numba>=0.56.2",
"numpy",
"nvtx>=0.2.1",
"packaging",
"pandas>=1.0,<1.6.0dev0",
"protobuf==4.21",
"typing_extensions",
# Allow floating minor versions for Arrow.
"pyarrow==10",
"rmm==23.4.*",
"ptxcompiler",
"cubinlinker",
"cupy-cuda11x",
]
classifiers = [
"Intended Audience :: Developers",
"Topic :: Database",
"Topic :: Scientific/Engineering",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.10",
Comment on lines +53 to +54
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we have the python 3.9 classifier consistent across all .toml's? Either we drop it everywhere or keep it incase python 3.9 packages are making a comeback.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question. We removed it in rmm so I'm going to continue removing it everywhere else during this transition for now. We can readjust later if we want to.

]

[project.optional-dependencies]
test = [
"pytest",
"pytest-benchmark",
"pytest-xdist",
"hypothesis",
"mimesis>=4.1.0",
"fastavro>=0.22.9",
"python-snappy>=0.6.0",
"pyorc",
"msgpack",
"transformers==4.24.0",
"tzdata",
]

[project.urls]
Homepage = "https://github.com/rapidsai/cudf"
Documentation = "https://docs.rapids.ai/api/cudf/stable/"

[tool.setuptools]
license-files = ["LICENSE"]

[tool.isort]
line_length = 79
multi_line_output = 3
Expand Down
56 changes: 0 additions & 56 deletions python/cudf/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,64 +3,8 @@
from setuptools import find_packages
from skbuild import setup

install_requires = [
"cachetools",
"cuda-python>=11.7.1,<12.0",
"fsspec>=0.6.0",
"numba>=0.56.2",
"numpy",
"nvtx>=0.2.1",
"packaging",
"pandas>=1.0,<1.6.0dev0",
"protobuf==4.21",
"typing_extensions",
# Allow floating minor versions for Arrow.
"pyarrow==10",
"rmm==23.4.*",
"ptxcompiler",
"cubinlinker",
"cupy-cuda11x",
]

extras_require = {
"test": [
"pytest",
"pytest-benchmark",
"pytest-xdist",
"hypothesis",
"mimesis>=4.1.0",
"fastavro>=0.22.9",
"python-snappy>=0.6.0",
"pyorc",
"msgpack",
"transformers==4.24.0",
"tzdata",
]
}

setup(
name="cudf",
version="23.04.00",
description="cuDF - GPU Dataframe",
url="https://github.com/rapidsai/cudf",
author="NVIDIA Corporation",
license="Apache 2.0",
classifiers=[
"Intended Audience :: Developers",
"Topic :: Database",
"Topic :: Scientific/Engineering",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
],
include_package_data=True,
packages=find_packages(include=["cudf", "cudf.*"]),
package_data={
key: ["*.pxd"] for key in find_packages(include=["cudf._lib*"])
},
install_requires=install_requires,
extras_require=extras_require,
zip_safe=False,
)
1 change: 1 addition & 0 deletions python/custreamz/LICENSE
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../LICENSE
47 changes: 46 additions & 1 deletion python/custreamz/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,57 @@
# Copyright (c) 2021-2022, NVIDIA CORPORATION.

[build-system]

build-backend = "setuptools.build_meta"
requires = [
"wheel",
"setuptools",
]

[project]
name = "custreamz"
version = "23.04.00"
description = "cuStreamz - GPU Accelerated Streaming"
readme = { file = "README.md", content-type = "text/markdown" }
authors = [
{ name = "NVIDIA Corporation" },
]
license = { text = "Apache 2.0" }
requires-python = ">=3.8"
dependencies = [
"cudf",
"cudf_kafka",
]
classifiers = [
"Intended Audience :: Developers",
"Topic :: Streaming",
"Topic :: Scientific/Engineering",
"Topic :: Apache Kafka",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
]

[project.optional-dependencies]
test = [
"pytest",
"pytest-xdist",
]

[project.urls]
Homepage = "https://github.com/rapidsai/cudf"

[tool.setuptools]
license-files = ["LICENSE"]
zip-safe = false

[tools.setuptools.packages.find]
include = [
"custreamz",
"custreamz.*",
]

[tool.isort]
line_length = 79
multi_line_output = 3
Expand Down
30 changes: 2 additions & 28 deletions python/custreamz/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,5 @@
# Copyright (c) 2020-2023, NVIDIA CORPORATION.

from setuptools import find_packages, setup
from setuptools import setup

install_requires = ["cudf_kafka", "cudf"]

extras_require = {"test": ["pytest", "pytest-xdist"]}

setup(
name="custreamz",
version="23.04.00",
description="cuStreamz - GPU Accelerated Streaming",
url="https://github.com/rapidsai/cudf",
author="NVIDIA Corporation",
license="Apache 2.0",
classifiers=[
"Intended Audience :: Developers",
"Topic :: Streaming",
"Topic :: Scientific/Engineering",
"Topic :: Apache Kafka",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
],
packages=find_packages(include=["custreamz", "custreamz.*"]),
install_requires=install_requires,
extras_require=extras_require,
zip_safe=False,
)
setup()
50 changes: 48 additions & 2 deletions python/dask_cudf/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,58 @@
# Copyright (c) 2021-2022, NVIDIA CORPORATION.
# Copyright (c) 2021-2023, NVIDIA CORPORATION.

[build-system]

build-backend = "setuptools.build_meta"
requires = [
"wheel",
"setuptools",
]

[project]
name = "dask_cudf"
version = "23.04.00"
description = "Utilities for Dask and cuDF interactions"
readme = { file = "README.md", content-type = "text/markdown" }
authors = [
{ name = "NVIDIA Corporation" },
]
license = { text = "Apache 2.0" }
requires-python = ">=3.8"
dependencies = [
"dask>=2023.1.1",
"distributed>=2023.1.1",
"fsspec>=0.6.0",
"numpy",
"pandas>=1.0,<1.6.0dev0",
"cudf==23.4.*",
"cupy-cuda11x",
]
classifiers = [
"Intended Audience :: Developers",
"Topic :: Database",
"Topic :: Scientific/Engineering",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
]
dynamic = ["entry-points"]

[project.optional-dependencies]
test = [
"numpy",
"pandas>=1.0,<1.6.0dev0",
"pytest",
"pytest-xdist",
"numba>=0.56.2",
]

[project.urls]
Homepage = "https://github.com/rapidsai/cudf"

[tool.setuptools]
license-files = ["LICENSE"]

[tool.isort]
line_length = 79
multi_line_output = 3
Expand Down
5 changes: 0 additions & 5 deletions python/dask_cudf/setup.cfg
View file
Open in desktop

This file was deleted.

Loading