From ad4087c8655af9855156634dcc0f81640c9fd108 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 8 Jun 2023 16:06:00 +0800 Subject: [PATCH] feat: allow to include files in wheel .data directory (#182) --- docs/build_config.md | 23 ++++++++++++++++ src/pdm/backend/config.py | 15 ++++++++++- src/pdm/backend/hooks/base.py | 9 ++++++- src/pdm/backend/wheel.py | 27 +++++++++++++++++++ .../demo-package-include/pyproject.toml | 4 +++ .../demo-package-include/scripts/my_script.sh | 3 +++ tests/test_api.py | 16 +++++++---- 7 files changed, 90 insertions(+), 7 deletions(-) create mode 100755 tests/fixtures/projects/demo-package-include/scripts/my_script.sh diff --git a/docs/build_config.md b/docs/build_config.md index cf0159b..23fac4b 100644 --- a/docs/build_config.md +++ b/docs/build_config.md @@ -195,6 +195,29 @@ If neither `includes` and `excludes` is specified, the backend can determine the `*.pyc`, `__pycache__/` and `build/` are always excluded. +### Wheel data files + +You can include additional files that are not normally installed inside site-packages directory, with `tool.pdm.build.wheel-data` table: + +```toml +[tool.pdm.build.wheel-data] +# Install all files under scripts/ to the $prefix/bin directory +scripts = ["scripts/*"] +# Install all files under include/ to the $prefix/include directory recursively, keeping the directory structure +include = [{path = "include/**/*.h", relative-to = "include/"}] +``` + +The key is the name of the install scheme, and should be one of `scripts`, `purelib`, `platlib`, `include`, `platinclude` and `data`. +And each value should be a list of items, which may contain the following attributes: + +- `path`: The path pattern to match the files to be included. +- `relative-to`: if specified, the relative paths of the matched files will be calculated based on this directory, +otherwise the files will be flattened and installed directly under the scheme directory. + +In both attributes, you can use `${BUILD_DIR}` to refer to the build directory. + +These files will be packaged into the `{name}-{version}.data/{scheme}` directory in the wheel distribution. + ## Local build hooks You can specify a custom script to be executed before the build process, which can be used to generate files or modify the metadata. diff --git a/src/pdm/backend/config.py b/src/pdm/backend/config.py index e71e1ee..8c6119f 100644 --- a/src/pdm/backend/config.py +++ b/src/pdm/backend/config.py @@ -4,7 +4,7 @@ import os import sys from pathlib import Path -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from pdm.backend._vendor import tomli_w from pdm.backend._vendor.pyproject_metadata import ConfigurationError, StandardMetadata @@ -19,6 +19,14 @@ T = TypeVar("T") +if TYPE_CHECKING: + from typing import TypedDict + + DataSpecDict = TypedDict( + "DataSpecDict", {"path": str, "relative-to": str}, total=False + ) + DataSpec = DataSpecDict | str + class Config: """The project config object for pdm backend. @@ -257,3 +265,8 @@ def editable_backend(self) -> str: - path: the legacy .pth file method(default) """ return self.get("editable-backend", "path") + + @property + def wheel_data(self) -> dict[str, list[DataSpec]]: + """The wheel data configuration""" + return self.get("wheel-data", {}) diff --git a/src/pdm/backend/hooks/base.py b/src/pdm/backend/hooks/base.py index 5c3b49a..33f6a27 100644 --- a/src/pdm/backend/hooks/base.py +++ b/src/pdm/backend/hooks/base.py @@ -2,7 +2,7 @@ import dataclasses from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Iterable from pdm.backend.config import Config @@ -60,6 +60,13 @@ def ensure_build_dir(self) -> Path: (self.build_dir / ".gitignore").write_text("*\n") return self.build_dir + def expand_paths(self, path: str) -> Iterable[Path]: + plib_path = Path(path) + if plib_path.parts and plib_path.parts[0] == "${BUILD_DIR}": + return self.build_dir.glob(Path(*plib_path.parts[1:]).as_posix()) + + return self.root.glob(path) + class BuildHookInterface(Protocol): """The interface definition for build hooks. diff --git a/src/pdm/backend/wheel.py b/src/pdm/backend/wheel.py index 277be2a..1e0c783 100644 --- a/src/pdm/backend/wheel.py +++ b/src/pdm/backend/wheel.py @@ -28,6 +28,10 @@ to_filename, ) +SCHEME_NAMES = frozenset( + ["purelib", "platlib", "include", "platinclude", "scripts", "data"] +) + if sys.version_info < (3, 8): from importlib_metadata import version as get_version else: @@ -72,6 +76,13 @@ def __init__( super().__init__(location, config_settings) self.__tag: str | None = None + def scheme_path(self, name: str, relative: str) -> str: + if name not in SCHEME_NAMES: + raise ValueError( + f"Unknown scheme name {name!r}, must be one of {SCHEME_NAMES}" + ) + return f"{self.name_version}.data/{name}/{relative}" + def _get_platform_tags(self) -> tuple[str | None, str | None, str | None]: python_tag: str | None = None py_limited_api: str | None = None @@ -118,6 +129,22 @@ def get_files(self, context: Context) -> Iterable[tuple[str, Path]]: relpath = relpath[len(package_dir) + 1 :] yield relpath, path yield from self._get_metadata_files(context) + yield from self._get_wheel_data(context) + + def _get_wheel_data(self, context: Context) -> Iterable[tuple[str, Path]]: + for name, paths in context.config.build_config.wheel_data.items(): + for path in paths: + relative_to: str | None = None + if not isinstance(path, str): + relative_to = path.get("relative-to") + path = path["path"] + for child in context.expand_paths(path): + relpath = ( + child.relative_to(relative_to).as_posix() + if relative_to + else child.name + ) + yield self.scheme_path(name, relpath), child def build_artifact( self, context: Context, files: Iterable[tuple[str, Path]] diff --git a/tests/fixtures/projects/demo-package-include/pyproject.toml b/tests/fixtures/projects/demo-package-include/pyproject.toml index 74d81ca..f548cdc 100644 --- a/tests/fixtures/projects/demo-package-include/pyproject.toml +++ b/tests/fixtures/projects/demo-package-include/pyproject.toml @@ -28,3 +28,7 @@ includes = [ excludes = [ "my_package/*.json" ] +source-includes = ["scripts/"] + +[tool.pdm.build.wheel-data] +scripts = ["scripts/*"] diff --git a/tests/fixtures/projects/demo-package-include/scripts/my_script.sh b/tests/fixtures/projects/demo-package-include/scripts/my_script.sh new file mode 100755 index 0000000..46c0d0e --- /dev/null +++ b/tests/fixtures/projects/demo-package-include/scripts/my_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Hello world" diff --git a/tests/test_api.py b/tests/test_api.py index 5a13315..3b68f69 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -84,17 +84,23 @@ def test_build_package_include(tmp_path: Path) -> None: assert wheel_name == "demo_package-0.1.0-py3-none-any.whl" tar_names = get_tarball_names(tmp_path / sdist_name) - zip_names = get_wheel_names(tmp_path / wheel_name) assert "demo_package-0.1.0/my_package/__init__.py" in tar_names assert "demo_package-0.1.0/my_package/data.json" not in tar_names assert "demo_package-0.1.0/requirements.txt" in tar_names assert "demo_package-0.1.0/data_out.json" in tar_names - assert "my_package/__init__.py" in zip_names - assert "my_package/data.json" not in zip_names - assert "requirements.txt" in zip_names - assert "data_out.json" in zip_names + with zipfile.ZipFile(tmp_path / wheel_name) as zf: + zip_names = zf.namelist() + assert "my_package/__init__.py" in zip_names + assert "my_package/data.json" not in zip_names + assert "requirements.txt" in zip_names + assert "data_out.json" in zip_names + assert "demo_package-0.1.0.data/scripts/my_script.sh" in zip_names + if os.name != "nt": + info = zf.getinfo("demo_package-0.1.0.data/scripts/my_script.sh") + filemode = info.external_attr >> 16 + assert filemode & 0o111 def test_namespace_package_by_include(tmp_path: Path) -> None: