From 9a2b1f1a900704bf9bb3db39b09f5213c35c423d Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 4 Aug 2022 18:42:02 -0500 Subject: [PATCH 1/3] Add `[python].resolves_to_constraints_file` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/goals/lockfile.py | 47 +++++++- .../backend/python/goals/lockfile_test.py | 103 +++++++++++------- .../pants/backend/python/subsystems/setup.py | 62 +++++++++-- 3 files changed, 156 insertions(+), 56 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index dfc7e79bb79..332af726e3f 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -40,11 +40,19 @@ WrappedGenerateLockfile, ) from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest -from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent +from pants.engine.fs import ( + CreateDigest, + Digest, + DigestContents, + FileContent, + GlobMatchErrorBehavior, + MergeDigests, + PathGlobs, +) from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import AllTargets -from pants.engine.unions import UnionRule +from pants.engine.unions import UnionMembership, UnionRule from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -154,6 +162,7 @@ async def generate_lockfile( generate_lockfiles_subsystem: GenerateLockfilesSubsystem, python_repos: PythonRepos, python_setup: PythonSetup, + union_membership: UnionMembership, ) -> GenerateLockfileResult: if req.use_pex: pip_args_file = "__pip_args.txt" @@ -161,9 +170,38 @@ async def generate_lockfile( [f"--no-binary {pkg}" for pkg in python_setup.no_binary] + [f"--only-binary {pkg}" for pkg in python_setup.only_binary] ) - pip_args_file_digest = await Get( + pip_args_digest_get = Get( Digest, CreateDigest([FileContent(pip_args_file, pip_args_file_content.encode())]) ) + + all_tool_resolve_names = tuple( + sentinel.resolve_name for sentinel in union_membership.get(GenerateToolLockfileSentinel) + ) + constraints_file_path = python_setup.resolves_to_constraints_file( + all_tool_resolve_names + ).get(req.resolve_name) + constraints_digest_get = Get( + Digest, + PathGlobs( + [constraints_file_path] if constraints_file_path else [], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=softwrap( + f""" + the option `[python].resolves_to_constraints_file` for the resolve + '{req.resolve_name}' + """ + ), + ), + ) + constraints_args = ( + (f"--constraints={constraints_file_path}",) if constraints_file_path else () + ) + + pip_args_digest, constraints_digest = await MultiGet( + pip_args_digest_get, constraints_digest_get + ) + input_digest = await Get(Digest, MergeDigests([pip_args_digest, constraints_digest])) + header_delimiter = "//" result = await Get( ProcessResult, @@ -194,12 +232,13 @@ async def generate_lockfile( "--indent=2", "-r", pip_args_file, + *constraints_args, *python_repos.pex_args, *python_setup.manylinux_pex_args, *req.interpreter_constraints.generate_pex_arg_list(), *req.requirements, ), - additional_input_digest=pip_args_file_digest, + additional_input_digest=input_digest, output_files=("lock.json",), description=f"Generate lockfile for {req.resolve_name}", # Instead of caching lockfile generation with LMDB, we instead use the invalidation diff --git a/src/python/pants/backend/python/goals/lockfile_test.py b/src/python/pants/backend/python/goals/lockfile_test.py index 1a9c750ae1b..aef7929ae0c 100644 --- a/src/python/pants/backend/python/goals/lockfile_test.py +++ b/src/python/pants/backend/python/goals/lockfile_test.py @@ -26,12 +26,27 @@ from pants.util.strutil import strip_prefix -def _generate(*, rule_runner: RuleRunner, use_pex: bool) -> str: +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *lockfile_rules(), + *pex.rules(), + QueryRule(GenerateLockfileResult, [GeneratePythonLockfile]), + ] + ) + rule_runner.set_options([], env_inherit=PYTHON_BOOTSTRAP_ENV) + return rule_runner + + +def _generate( + *, rule_runner: RuleRunner, use_pex: bool, ansicolors_version: str = "==1.1.8" +) -> str: result = rule_runner.request( GenerateLockfileResult, [ GeneratePythonLockfile( - requirements=FrozenOrderedSet(["ansicolors==1.1.8"]), + requirements=FrozenOrderedSet([f"ansicolors{ansicolors_version}"]), interpreter_constraints=InterpreterConstraints(), resolve_name="test", lockfile_dest="test.lock", @@ -41,19 +56,32 @@ def _generate(*, rule_runner: RuleRunner, use_pex: bool) -> str: ) digest_contents = rule_runner.request(DigestContents, [result.digest]) assert len(digest_contents) == 1 - return digest_contents[0].content.decode() + content = digest_contents[0].content.decode() + if not use_pex: + return content - -def test_poetry_lockfile_generation() -> None: - rule_runner = RuleRunner( - rules=[ - *lockfile_rules(), - *pex.rules(), - QueryRule(GenerateLockfileResult, [GeneratePythonLockfile]), - ] + pex_header = dedent( + f"""\ + // This lockfile was autogenerated by Pants. To regenerate, run: + // + // ./pants generate-lockfiles --resolve=test + // + // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- + // {{ + // "version": 2, + // "valid_for_interpreter_constraints": [], + // "generated_with_requirements": [ + // "ansicolors{ansicolors_version}" + // ] + // }} + // --- END PANTS LOCKFILE METADATA --- + """ ) - rule_runner.set_options([], env_inherit=PYTHON_BOOTSTRAP_ENV) + assert content.startswith(pex_header) + return strip_prefix(content, pex_header) + +def test_poetry_lockfile_generation(rule_runner: RuleRunner) -> None: poetry_lock = _generate(rule_runner=rule_runner, use_pex=False) assert poetry_lock.startswith("# This lockfile was autogenerated by Pants.") assert poetry_lock.rstrip().endswith( @@ -69,14 +97,9 @@ def test_poetry_lockfile_generation() -> None: @pytest.mark.parametrize( ("no_binary", "only_binary"), ((False, False), (False, True), (True, False)) ) -def test_pex_lockfile_generation(no_binary: bool, only_binary: bool) -> None: - rule_runner = RuleRunner( - rules=[ - *lockfile_rules(), - *pex.rules(), - QueryRule(GenerateLockfileResult, [GeneratePythonLockfile]), - ] - ) +def test_pex_lockfile_generation( + rule_runner: RuleRunner, no_binary: bool, only_binary: bool +) -> None: args = [] if no_binary: args.append("--python-no-binary=ansicolors") @@ -84,26 +107,7 @@ def test_pex_lockfile_generation(no_binary: bool, only_binary: bool) -> None: args.append("--python-only-binary=ansicolors") rule_runner.set_options(args, env_inherit=PYTHON_BOOTSTRAP_ENV) - pex_header = dedent( - """\ - // This lockfile was autogenerated by Pants. To regenerate, run: - // - // ./pants generate-lockfiles --resolve=test - // - // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- - // { - // "version": 2, - // "valid_for_interpreter_constraints": [], - // "generated_with_requirements": [ - // "ansicolors==1.1.8" - // ] - // } - // --- END PANTS LOCKFILE METADATA --- - """ - ) - pex_lock = _generate(rule_runner=rule_runner, use_pex=True) - assert pex_lock.startswith(pex_header) - lock_entry = json.loads(strip_prefix(pex_lock, pex_header)) + lock_entry = json.loads(_generate(rule_runner=rule_runner, use_pex=True)) reqs = lock_entry["locked_resolves"][0]["locked_requirements"] assert len(reqs) == 1 assert reqs[0]["project_name"] == "ansicolors" @@ -142,6 +146,25 @@ def test_pex_lockfile_generation(no_binary: bool, only_binary: bool) -> None: assert artifacts == [wheel] +def test_constraints_file(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"constraints.txt": "ansicolors==1.1.7"}) + rule_runner.set_options( + [ + "--python-resolves={'test': 'foo.lock'}", + "--python-resolves-to-constraints-file={'test': 'constraints.txt'}", + ], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + + lock_entry = json.loads( + _generate(rule_runner=rule_runner, use_pex=True, ansicolors_version=">=1.0") + ) + reqs = lock_entry["locked_resolves"][0]["locked_requirements"] + assert len(reqs) == 1 + assert reqs[0]["project_name"] == "ansicolors" + assert reqs[0]["version"] == "1.1.7" + + def test_multiple_resolves() -> None: rule_runner = RuleRunner( rules=[ diff --git a/src/python/pants/backend/python/subsystems/setup.py b/src/python/pants/backend/python/subsystems/setup.py index 257bdfbabb0..51518dae4eb 100644 --- a/src/python/pants/backend/python/subsystems/setup.py +++ b/src/python/pants/backend/python/subsystems/setup.py @@ -8,6 +8,7 @@ import os from typing import Iterable, Iterator, Optional, cast +from pants.core.goals.generate_lockfiles import UnrecognizedResolveNamesError from pants.option.option_types import ( BoolOption, DictOption, @@ -18,7 +19,7 @@ ) from pants.option.subsystem import Subsystem from pants.util.docutil import bin_name, doc_url -from pants.util.memo import memoized_property +from pants.util.memo import memoized_method, memoized_property from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @@ -204,7 +205,32 @@ class PythonSetup(Subsystem): using a resolve whose interpreter constraints are set to ['==3.7.*'], then Pants will error explaining the incompatibility. - The keys must be defined as resolves in `[python].resolves`. + The keys must be defined as resolves in `[python].resolves`. To change the interpreter + constraints for tool lockfiles, change `[tool].interpreter_constraints`, e.g. + `[black].interpreter_constraints`; if the tool does not have that option, it determines + its interpreter constraints from your user code. + """ + ), + advanced=True, + ) + _resolves_to_constraints_file = DictOption[str]( + help=softwrap( + """ + When generating a resolve's lockfile, use a constraints file to pin the version of + certain requirements. This is particularly useful to pin the versions of transitive + dependencies of your direct requirements. + + See https://pip.pypa.io/en/stable/user_guide/#constraints-files for more information on + the format of constraint files and how constraints are applied in Pex and pip. + + Expects a dictionary of resolve names from `[python].resolves` and Python tools (e.g. + `black` and `pytest`) to file paths for + constraints files. For example, + `{'data-science': '3rdparty/data-science-constraints.txt'}`. + If a resolve is not set in the dictionary, it will not use a constraints file. + + Note: Only takes effect if you use Pex lockfiles. Use the default + `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal. """ ), advanced=True, @@ -491,21 +517,33 @@ def generate_lockfiles_with_pex(self) -> bool: @memoized_property def resolves_to_interpreter_constraints(self) -> dict[str, tuple[str, ...]]: result = {} + unrecognized_resolves = [] for resolve, ics in self._resolves_to_interpreter_constraints.items(): if resolve not in self.resolves: - raise KeyError( - softwrap( - f""" - Unrecognized resolve name in the option - `[python].resolves_to_interpreter_constraints`: {resolve}. Each - key must be one of the keys in `[python].resolves`: - {sorted(self.resolves.keys())} - """ - ) - ) + unrecognized_resolves.append(resolve) result[resolve] = tuple(ics) + if unrecognized_resolves: + raise UnrecognizedResolveNamesError( + unrecognized_resolves, + self.resolves.keys(), + description_of_origin="the option `[python].resolves_to_interpreter_constraints`", + ) return result + @memoized_method + def resolves_to_constraints_file( + self, all_tool_resolve_names: tuple[str, ...] + ) -> dict[str, str]: + all_valid_resolves = {*self.resolves, *all_tool_resolve_names} + unrecognized_resolves = set(self._resolves_to_constraints_file.keys()) - all_valid_resolves + if unrecognized_resolves: + raise UnrecognizedResolveNamesError( + sorted(unrecognized_resolves), + all_valid_resolves, + description_of_origin="the option `[python].resolves_to_constraints_file`", + ) + return self._resolves_to_constraints_file + def resolve_all_constraints_was_set_explicitly(self) -> bool: return not self.options.is_default("resolve_all_constraints") From 1c24cc18db9f12b5d729f26dfdae2e3a4ac2f689 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 5 Aug 2022 23:40:17 -0500 Subject: [PATCH 2/3] Track constraints file in lockfile metadata [ci skip-rust] [ci skip-build-wheels] --- .../pants/backend/python/goals/lockfile.py | 107 +++++++++--------- .../backend/python/goals/lockfile_test.py | 21 +++- .../python/util_rules/lockfile_metadata.py | 77 ++++++++++++- .../util_rules/lockfile_metadata_test.py | 52 +++++++-- .../pants/backend/python/util_rules/pex.py | 20 +++- .../python/util_rules/pex_requirements.py | 99 +++++++++++++++- .../util_rules/pex_requirements_test.py | 56 ++++++--- .../backend/python/util_rules/pex_test.py | 4 +- .../core/goals/update_build_files_test.py | 9 +- .../core/util_rules/lockfile_metadata.py | 3 +- 10 files changed, 351 insertions(+), 97 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 332af726e3f..ea1f076fcee 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -27,7 +27,11 @@ from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_cli import PexCliProcess -from pants.backend.python.util_rules.pex_requirements import PexRequirements +from pants.backend.python.util_rules.pex_requirements import ( + PexRequirements, + ResolvePexConfig, + ResolvePexConfigRequest, +) from pants.core.goals.generate_lockfiles import ( GenerateLockfile, GenerateLockfileResult, @@ -40,19 +44,12 @@ WrappedGenerateLockfile, ) from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest -from pants.engine.fs import ( - CreateDigest, - Digest, - DigestContents, - FileContent, - GlobMatchErrorBehavior, - MergeDigests, - PathGlobs, -) +from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent, MergeDigests +from pants.engine.internals.native_engine import FileDigest from pants.engine.process import ProcessCacheScope, ProcessResult -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper from pants.engine.target import AllTargets -from pants.engine.unions import UnionMembership, UnionRule +from pants.engine.unions import UnionRule from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -155,52 +152,55 @@ def warn_python_repos(option: str) -> None: return MaybeWarnPythonRepos() -@rule(desc="Generate Python lockfile", level=LogLevel.DEBUG) -async def generate_lockfile( - req: GeneratePythonLockfile, - poetry_subsystem: PoetrySubsystem, - generate_lockfiles_subsystem: GenerateLockfilesSubsystem, - python_repos: PythonRepos, - python_setup: PythonSetup, - union_membership: UnionMembership, -) -> GenerateLockfileResult: - if req.use_pex: +@rule_helper +async def _setup_pip_args_and_constraints_file( + python_setup: PythonSetup, *, resolve_name: str +) -> tuple[list[str], Digest, FileDigest | None]: + extra_args = [] + extra_digests = [] + constraints_file_digest: None | FileDigest = None + + if python_setup.no_binary or python_setup.only_binary: pip_args_file = "__pip_args.txt" + extra_args.extend(["-r", pip_args_file]) pip_args_file_content = "\n".join( [f"--no-binary {pkg}" for pkg in python_setup.no_binary] + [f"--only-binary {pkg}" for pkg in python_setup.only_binary] ) - pip_args_digest_get = Get( + pip_args_digest = await Get( Digest, CreateDigest([FileContent(pip_args_file, pip_args_file_content.encode())]) ) + extra_digests.append(pip_args_digest) - all_tool_resolve_names = tuple( - sentinel.resolve_name for sentinel in union_membership.get(GenerateToolLockfileSentinel) - ) - constraints_file_path = python_setup.resolves_to_constraints_file( - all_tool_resolve_names - ).get(req.resolve_name) - constraints_digest_get = Get( - Digest, - PathGlobs( - [constraints_file_path] if constraints_file_path else [], - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=softwrap( - f""" - the option `[python].resolves_to_constraints_file` for the resolve - '{req.resolve_name}' - """ - ), - ), - ) - constraints_args = ( - (f"--constraints={constraints_file_path}",) if constraints_file_path else () - ) + resolve_config = await Get(ResolvePexConfig, ResolvePexConfigRequest(resolve_name)) + if resolve_config.constraints_file: + _constraints_file_entry = resolve_config.constraints_file[1] + extra_args.append(f"--constraints={_constraints_file_entry.path}") + constraints_file_digest = _constraints_file_entry.file_digest + extra_digests.append(resolve_config.constraints_file[0]) - pip_args_digest, constraints_digest = await MultiGet( - pip_args_digest_get, constraints_digest_get - ) - input_digest = await Get(Digest, MergeDigests([pip_args_digest, constraints_digest])) + input_digest = await Get(Digest, MergeDigests(extra_digests)) + return extra_args, input_digest, constraints_file_digest + + +@rule(desc="Generate Python lockfile", level=LogLevel.DEBUG) +async def generate_lockfile( + req: GeneratePythonLockfile, + poetry_subsystem: PoetrySubsystem, + generate_lockfiles_subsystem: GenerateLockfilesSubsystem, + python_repos: PythonRepos, + python_setup: PythonSetup, +) -> GenerateLockfileResult: + constraints_file_hash: str | None = None + + if req.use_pex: + ( + extra_args, + input_digest, + constraints_file_digest, + ) = await _setup_pip_args_and_constraints_file(python_setup, resolve_name=req.resolve_name) + if constraints_file_digest: + constraints_file_hash = constraints_file_digest.fingerprint header_delimiter = "//" result = await Get( @@ -230,9 +230,7 @@ async def generate_lockfile( "mac", # This makes diffs more readable when lockfiles change. "--indent=2", - "-r", - pip_args_file, - *constraints_args, + *extra_args, *python_repos.pex_args, *python_setup.manylinux_pex_args, *req.interpreter_constraints.generate_pex_arg_list(), @@ -303,8 +301,9 @@ async def generate_lockfile( initial_lockfile_digest_contents = await Get(DigestContents, Digest, result.output_digest) # TODO(#12314) Improve error message on `Requirement.parse` metadata = PythonLockfileMetadata.new( - req.interpreter_constraints, - {PipRequirement.parse(i) for i in req.requirements}, + valid_for_interpreter_constraints=req.interpreter_constraints, + requirements={PipRequirement.parse(i) for i in req.requirements}, + constraints_file_hash=constraints_file_hash, ) lockfile_with_header = metadata.add_header_to_lockfile( initial_lockfile_digest_contents[0].content, diff --git a/src/python/pants/backend/python/goals/lockfile_test.py b/src/python/pants/backend/python/goals/lockfile_test.py index aef7929ae0c..5365e6bf72a 100644 --- a/src/python/pants/backend/python/goals/lockfile_test.py +++ b/src/python/pants/backend/python/goals/lockfile_test.py @@ -40,7 +40,11 @@ def rule_runner() -> RuleRunner: def _generate( - *, rule_runner: RuleRunner, use_pex: bool, ansicolors_version: str = "==1.1.8" + *, + rule_runner: RuleRunner, + use_pex: bool, + ansicolors_version: str = "==1.1.8", + constraints_file_hash: str | None = None, ) -> str: result = rule_runner.request( GenerateLockfileResult, @@ -60,6 +64,7 @@ def _generate( if not use_pex: return content + constraints_file_hash_str = f'"{constraints_file_hash}"' if constraints_file_hash else "null" pex_header = dedent( f"""\ // This lockfile was autogenerated by Pants. To regenerate, run: @@ -68,11 +73,12 @@ def _generate( // // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- // {{ - // "version": 2, + // "version": 3, // "valid_for_interpreter_constraints": [], // "generated_with_requirements": [ // "ansicolors{ansicolors_version}" - // ] + // ], + // "constraints_file_hash": {constraints_file_hash_str} // }} // --- END PANTS LOCKFILE METADATA --- """ @@ -157,7 +163,14 @@ def test_constraints_file(rule_runner: RuleRunner) -> None: ) lock_entry = json.loads( - _generate(rule_runner=rule_runner, use_pex=True, ansicolors_version=">=1.0") + _generate( + rule_runner=rule_runner, + use_pex=True, + ansicolors_version=">=1.0", + constraints_file_hash=( + "1999760ce9dd0f82847def308992e3345592fc9e77a937c1e9bbb78a42ae3943" + ), + ) ) reqs = lock_entry["locked_resolves"][0]["locked_requirements"] assert len(reqs) == 1 diff --git a/src/python/pants/backend/python/util_rules/lockfile_metadata.py b/src/python/pants/backend/python/util_rules/lockfile_metadata.py index 18701d9c857..9092bee8086 100644 --- a/src/python/pants/backend/python/util_rules/lockfile_metadata.py +++ b/src/python/pants/backend/python/util_rules/lockfile_metadata.py @@ -24,6 +24,7 @@ class InvalidPythonLockfileReason(Enum): INVALIDATION_DIGEST_MISMATCH = "invalidation_digest_mismatch" INTERPRETER_CONSTRAINTS_MISMATCH = "interpreter_constraints_mismatch" REQUIREMENTS_MISMATCH = "requirements_mismatch" + CONSTRAINTS_FILE_MISMATCH = "constraints_file_mismatch" @dataclass(frozen=True) @@ -35,8 +36,10 @@ class PythonLockfileMetadata(LockfileMetadata): @staticmethod def new( + *, valid_for_interpreter_constraints: InterpreterConstraints, requirements: set[PipRequirement], + constraints_file_hash: str | None, ) -> PythonLockfileMetadata: """Call the most recent version of the `LockfileMetadata` class to construct a concrete instance. @@ -46,7 +49,9 @@ def new( writing, while still allowing us to support _reading_ older, deprecated metadata versions. """ - return PythonLockfileMetadataV2(valid_for_interpreter_constraints, requirements) + return PythonLockfileMetadataV3( + valid_for_interpreter_constraints, requirements, constraints_file_hash + ) @classmethod def additional_header_attrs(cls, instance: LockfileMetadata) -> dict[Any, Any]: @@ -65,6 +70,7 @@ def is_valid_for( user_interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str], user_requirements: Iterable[PipRequirement], + constraints_file_path_and_hash: tuple[str, str] | None, ) -> LockfileMetadataValidation: """Returns Truthy if this `PythonLockfileMetadata` can be used in the current execution context.""" @@ -106,7 +112,9 @@ def is_valid_for( expected_invalidation_digest: str | None, user_interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str], - user_requirements: Iterable[PipRequirement], # User requirements are not used by V1 + # Everything below is not used by v1. + user_requirements: Iterable[PipRequirement], + constraints_file_path_and_hash: tuple[str, str] | None, ) -> LockfileMetadataValidation: failure_reasons: set[InvalidPythonLockfileReason] = set() @@ -172,10 +180,12 @@ def is_valid_for( self, *, is_tool: bool, - expected_invalidation_digest: str | None, # Validation digests are not used by V2. + expected_invalidation_digest: str | None, # Not used by V2. user_interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str], user_requirements: Iterable[PipRequirement], + # Everything below is not used by V2. + constraints_file_path_and_hash: tuple[str, str] | None, ) -> LockfileMetadataValidation: failure_reasons = set() @@ -193,3 +203,64 @@ def is_valid_for( failure_reasons.add(InvalidPythonLockfileReason.INTERPRETER_CONSTRAINTS_MISMATCH) return LockfileMetadataValidation(failure_reasons) + + +@_python_lockfile_metadata(3) +@dataclass(frozen=True) +class PythonLockfileMetadataV3(PythonLockfileMetadataV2): + """Lockfile version that considers constraints files.""" + + constraints_file_hash: str | None + + @classmethod + def _from_json_dict( + cls: type[PythonLockfileMetadataV3], + json_dict: dict[Any, Any], + lockfile_description: str, + error_suffix: str, + ) -> PythonLockfileMetadataV3: + v2_metadata = super()._from_json_dict(json_dict, lockfile_description, error_suffix) + metadata = _get_metadata(json_dict, lockfile_description, error_suffix) + constraints_file_hash = metadata( + "constraints_file_hash", str, lambda x: x # type: ignore[no-any-return] + ) + return PythonLockfileMetadataV3( + valid_for_interpreter_constraints=v2_metadata.valid_for_interpreter_constraints, + requirements=v2_metadata.requirements, + constraints_file_hash=constraints_file_hash, + ) + + @classmethod + def additional_header_attrs(cls, instance: LockfileMetadata) -> dict[Any, Any]: + instance = cast(PythonLockfileMetadataV3, instance) + return {"constraints_file_hash": instance.constraints_file_hash} + + def is_valid_for( + self, + *, + is_tool: bool, + expected_invalidation_digest: str | None, # Validation digests are not used by V2. + user_interpreter_constraints: InterpreterConstraints, + interpreter_universe: Iterable[str], + user_requirements: Iterable[PipRequirement], + constraints_file_path_and_hash: tuple[str, str] | None, + ) -> LockfileMetadataValidation: + failure_reasons = ( + super() + .is_valid_for( + is_tool=is_tool, + expected_invalidation_digest=expected_invalidation_digest, + user_interpreter_constraints=user_interpreter_constraints, + interpreter_universe=interpreter_universe, + user_requirements=user_requirements, + constraints_file_path_and_hash=constraints_file_path_and_hash, + ) + .failure_reasons + ) + + provided_constraints_file_hash = ( + constraints_file_path_and_hash[1] if constraints_file_path_and_hash else None + ) + if provided_constraints_file_hash != self.constraints_file_hash: + failure_reasons.add(InvalidPythonLockfileReason.CONSTRAINTS_FILE_MISMATCH) + return LockfileMetadataValidation(failure_reasons) diff --git a/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py b/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py index 670543fa620..39a942998b3 100644 --- a/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py +++ b/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py @@ -15,6 +15,7 @@ PythonLockfileMetadata, PythonLockfileMetadataV1, PythonLockfileMetadataV2, + PythonLockfileMetadataV3, ) from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest @@ -27,8 +28,11 @@ def reqset(*a) -> set[PipRequirement]: def test_metadata_header_round_trip() -> None: input_metadata = PythonLockfileMetadata.new( - InterpreterConstraints(["CPython==2.7.*", "PyPy", "CPython>=3.6,<4,!=3.7.*"]), - reqset("ansicolors==0.1.0"), + valid_for_interpreter_constraints=InterpreterConstraints( + ["CPython==2.7.*", "PyPy", "CPython>=3.6,<4,!=3.7.*"] + ), + requirements=reqset("ansicolors==0.1.0"), + constraints_file_hash="abc", ) serialized_lockfile = input_metadata.add_header_to_lockfile( b"req1==1.0", regenerate_command="./pants lock", delimeter="#" @@ -51,13 +55,14 @@ def test_add_header_to_lockfile() -> None: # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 2, +# "version": 3, # "valid_for_interpreter_constraints": [ # "CPython>=3.7" # ], # "generated_with_requirements": [ # "ansicolors==0.1.0" -# ] +# ], +# "constraints_file_hash": null # } # --- END PANTS LOCKFILE METADATA --- dave==3.1.4 \\ @@ -68,7 +73,9 @@ def line_by_line(b: bytes) -> list[bytes]: return [i for i in (j.strip() for j in b.splitlines()) if i] metadata = PythonLockfileMetadata.new( - InterpreterConstraints([">=3.7"]), reqset("ansicolors==0.1.0") + valid_for_interpreter_constraints=InterpreterConstraints([">=3.7"]), + requirements=reqset("ansicolors==0.1.0"), + constraints_file_hash=None, ) result = metadata.add_header_to_lockfile( input_lockfile, regenerate_command="./pants lock", delimeter="#" @@ -151,6 +158,7 @@ def test_is_valid_for_v1(user_digest, expected_digest, user_ic, expected_ic, mat user_interpreter_constraints=InterpreterConstraints(user_ic), interpreter_universe=INTERPRETER_UNIVERSE, user_requirements=set(), + constraints_file_path_and_hash=None, ) ) == matches @@ -212,7 +220,7 @@ def test_is_valid_for_v1(user_digest, expected_digest, user_ic, expected_ic, mat ], ], ) -def test_is_valid_for_v2( +def test_is_valid_for_interpreter_constraints_and_requirements( is_tool: bool, user_ics: list[str], lock_ics: list[str], @@ -220,12 +228,34 @@ def test_is_valid_for_v2( lock_reqs: list[str], expected: list[InvalidPythonLockfileReason], ) -> None: - m = PythonLockfileMetadataV2(InterpreterConstraints(lock_ics), reqset(*lock_reqs)) - result = m.is_valid_for( + """This logic is used by V2 and newer.""" + for m in [ + PythonLockfileMetadataV2(InterpreterConstraints(lock_ics), reqset(*lock_reqs)), + PythonLockfileMetadataV3( + InterpreterConstraints(lock_ics), reqset(*lock_reqs), constraints_file_hash=None + ), + ]: + result = m.is_valid_for( + is_tool=is_tool, + expected_invalidation_digest="", + user_interpreter_constraints=InterpreterConstraints(user_ics), + interpreter_universe=INTERPRETER_UNIVERSE, + user_requirements=reqset(*user_reqs), + constraints_file_path_and_hash=None, + ) + assert result.failure_reasons == set(expected) + + +@pytest.mark.parametrize("is_tool", [True, False]) +def test_is_valid_for_constraints_file_hash(is_tool: bool) -> None: + result = PythonLockfileMetadataV3( + InterpreterConstraints([]), reqset(), constraints_file_hash="abc" + ).is_valid_for( is_tool=is_tool, expected_invalidation_digest="", - user_interpreter_constraints=InterpreterConstraints(user_ics), + user_interpreter_constraints=InterpreterConstraints([]), interpreter_universe=INTERPRETER_UNIVERSE, - user_requirements=reqset(*user_reqs), + user_requirements=reqset(), + constraints_file_path_and_hash=("c.txt", "xyz"), ) - assert result.failure_reasons == set(expected) + assert result.failure_reasons == {InvalidPythonLockfileReason.CONSTRAINTS_FILE_MISMATCH} diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index e9bc17e2142..1f0c18db16b 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -43,7 +43,11 @@ from pants.backend.python.util_rules.pex_requirements import ( PexRequirements as PexRequirements, # Explicit re-export. ) -from pants.backend.python.util_rules.pex_requirements import validate_metadata +from pants.backend.python.util_rules.pex_requirements import ( + ResolvePexConfig, + ResolvePexConfigRequest, + validate_metadata, +) from pants.core.target_types import FileSourceField from pants.core.util_rules.system_binaries import BashBinary from pants.engine.addresses import UnparsedAddressInputs @@ -415,7 +419,13 @@ async def build_pex( pex_lock_resolver_args = [*python_repos.pex_args] pip_resolver_args = [*python_repos.pex_args, "--resolver-version", "pip-2020-resolver"] if isinstance(request.requirements, EntireLockfile): - lockfile = await Get(LoadedLockfile, LoadedLockfileRequest(request.requirements.lockfile)) + lockfile, resolve_config = await MultiGet( + Get(LoadedLockfile, LoadedLockfileRequest(request.requirements.lockfile)), + Get( + ResolvePexConfig, + ResolvePexConfigRequest(request.requirements.lockfile.resolve_name), + ), + ) concurrency_available = lockfile.requirement_estimate requirements_digests.append(lockfile.lockfile_digest) if lockfile.is_pex_native: @@ -432,6 +442,7 @@ async def build_pex( lockfile.original_lockfile, request.requirements.complete_req_strings, python_setup, + constraints_file_path_and_hash=resolve_config.constraints_file_path_and_hash, ) else: # TODO: This is not the best heuristic for available concurrency, since the @@ -446,6 +457,10 @@ async def build_pex( requirements_digests.append(repository_pex.digest) elif isinstance(request.requirements.from_superset, LoadedLockfile): loaded_lockfile = request.requirements.from_superset + resolve_config = await Get( + ResolvePexConfig, + ResolvePexConfigRequest(loaded_lockfile.original_lockfile.resolve_name), + ) # NB: This is also validated in the constructor. assert loaded_lockfile.is_pex_native if request.requirements.req_strings: @@ -460,6 +475,7 @@ async def build_pex( loaded_lockfile.original_lockfile, request.requirements.req_strings, python_setup, + constraints_file_path_and_hash=resolve_config.constraints_file_path_and_hash, ) else: assert request.requirements.from_superset is None diff --git a/src/python/pants/backend/python/util_rules/pex_requirements.py b/src/python/pants/backend/python/util_rules/pex_requirements.py index d4bbb56fd46..b87d3d42e83 100644 --- a/src/python/pants/backend/python/util_rules/pex_requirements.py +++ b/src/python/pants/backend/python/util_rules/pex_requirements.py @@ -16,16 +16,20 @@ PythonLockfileMetadata, PythonLockfileMetadataV2, ) +from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.util_rules.lockfile_metadata import InvalidLockfileError, LockfileMetadataValidation from pants.engine.fs import ( CreateDigest, Digest, DigestContents, + DigestEntries, FileContent, + FileEntry, GlobMatchErrorBehavior, PathGlobs, ) -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionMembership from pants.util.docutil import bin_name, doc_url from pants.util.meta import frozen_after_init from pants.util.ordered_set import FrozenOrderedSet @@ -269,6 +273,74 @@ def __bool__(self) -> bool: return bool(self.req_strings) +@dataclass(frozen=True) +class ResolvePexConfig: + """Configuration from `[python]` that impacts how the resolve is created.""" + + constraints_file: tuple[Digest, FileEntry] | None + + @property + def constraints_file_path_and_hash(self) -> tuple[str, str] | None: + if self.constraints_file is None: + return None + file_entry = self.constraints_file[1] + return file_entry.path, file_entry.file_digest.fingerprint + + +@dataclass(frozen=True) +class ResolvePexConfigRequest: + """Find all configuration from `[python]` that impacts how the resolve is created.""" + + resolve_name: str + + +@rule +async def determine_resolve_pex_config( + request: ResolvePexConfigRequest, python_setup: PythonSetup, union_membership: UnionMembership +) -> ResolvePexConfig: + all_tool_resolve_names = tuple( + sentinel.resolve_name for sentinel in union_membership.get(GenerateToolLockfileSentinel) + ) + + constraints_file: tuple[Digest, FileEntry] | None = None + _constraints_file_path = python_setup.resolves_to_constraints_file(all_tool_resolve_names).get( + request.resolve_name + ) + if _constraints_file_path: + _constraints_origin = softwrap( + f""" + the option `[python].resolves_to_constraints_file` for the resolve + '{request.resolve_name}' + """ + ) + _constraints_path_globs = PathGlobs( + [_constraints_file_path] if _constraints_file_path else [], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=_constraints_origin, + ) + _constraints_digest, _constraints_digest_entries = await MultiGet( + Get(Digest, PathGlobs, _constraints_path_globs), + Get(DigestEntries, PathGlobs, _constraints_path_globs), + ) + + if len(_constraints_digest_entries) != 1: + raise ValueError( + softwrap( + f""" + Expected only one file from {_constraints_origin}, but matched: + {_constraints_digest_entries} + + Did you use a glob like `*`? + """ + ) + ) + _constraints_file_entry = next(iter(_constraints_digest_entries)) + assert isinstance(_constraints_file_entry, FileEntry) + constraints_file = (_constraints_digest, _constraints_file_entry) + + return ResolvePexConfig(constraints_file=constraints_file) + + def should_validate_metadata( lockfile: Lockfile | LockfileContent, python_setup: PythonSetup, @@ -285,6 +357,8 @@ def validate_metadata( lockfile: Lockfile | LockfileContent, consumed_req_strings: Iterable[str], python_setup: PythonSetup, + *, + constraints_file_path_and_hash: tuple[str, str] | None, ) -> None: """Given interpreter constraints and requirements to be consumed, validate lockfile metadata.""" @@ -296,6 +370,7 @@ def validate_metadata( user_interpreter_constraints=interpreter_constraints, interpreter_universe=python_setup.interpreter_versions_universe, user_requirements=user_requirements, + constraints_file_path_and_hash=constraints_file_path_and_hash, ) if validation: return @@ -306,6 +381,9 @@ def validate_metadata( lockfile=lockfile, user_interpreter_constraints=interpreter_constraints, user_requirements=user_requirements, + maybe_constraints_file_path=constraints_file_path_and_hash[0] + if constraints_file_path_and_hash + else None, ) is_tool = isinstance(lockfile, (ToolCustomLockfile, ToolDefaultLockfile)) msg_iter = ( @@ -319,6 +397,15 @@ def validate_metadata( logger.warning("%s", msg) +def _stale_constraints_file_error(file_path: str) -> str: + return softwrap( + f""" + - The constraints file at {file_path} has changed from when the lockfile was generated. + (Constraints files are set via the option `[python].resolves_to_constraints_file`) + """ + ) + + def _invalid_tool_lockfile_error( metadata: PythonLockfileMetadata, validation: LockfileMetadataValidation, @@ -326,6 +413,7 @@ def _invalid_tool_lockfile_error( *, user_requirements: set[PipRequirement], user_interpreter_constraints: InterpreterConstraints, + maybe_constraints_file_path: str | None, ) -> Iterator[str]: tool_name = lockfile.resolve_name @@ -399,6 +487,10 @@ def _invalid_tool_lockfile_error( ) yield "\n\n" + if InvalidPythonLockfileReason.CONSTRAINTS_FILE_MISMATCH in validation.failure_reasons: + assert maybe_constraints_file_path is not None + yield _stale_constraints_file_error(maybe_constraints_file_path) + yield softwrap( f""" To regenerate your lockfile based on your current configuration, run @@ -420,6 +512,7 @@ def _invalid_user_lockfile_error( *, user_requirements: set[PipRequirement], user_interpreter_constraints: InterpreterConstraints, + maybe_constraints_file_path: str | None, ) -> Iterator[str]: yield "You are using the lockfile " yield f"at {lockfile.file_path} " if isinstance( @@ -474,6 +567,10 @@ def _invalid_user_lockfile_error( """ ) + "\n\n" + if InvalidPythonLockfileReason.CONSTRAINTS_FILE_MISMATCH in validation.failure_reasons: + assert maybe_constraints_file_path is not None + yield _stale_constraints_file_error(maybe_constraints_file_path) + yield "To regenerate your lockfile, " yield f"run `{bin_name()} generate-lockfiles --resolve={lockfile.resolve_name}`." if isinstance( lockfile, Lockfile diff --git a/src/python/pants/backend/python/util_rules/pex_requirements_test.py b/src/python/pants/backend/python/util_rules/pex_requirements_test.py index dff73de5e36..0e8b3679c99 100644 --- a/src/python/pants/backend/python/util_rules/pex_requirements_test.py +++ b/src/python/pants/backend/python/util_rules/pex_requirements_test.py @@ -10,7 +10,7 @@ from pants.backend.python.pip_requirement import PipRequirement from pants.backend.python.subsystems.setup import InvalidLockfileBehavior, PythonSetup from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadataV2 +from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadataV3 from pants.backend.python.util_rules.pex_requirements import ( Lockfile, ToolCustomLockfile, @@ -25,9 +25,10 @@ from pants.testutil.option_util import create_subsystem from pants.util.ordered_set import FrozenOrderedSet -METADATA = PythonLockfileMetadataV2( +METADATA = PythonLockfileMetadataV3( InterpreterConstraints(["==3.8.*"]), {PipRequirement.parse("ansicolors"), PipRequirement.parse("requests")}, + constraints_file_hash="abc", ) @@ -77,28 +78,37 @@ def test_invalid_lockfile_behavior_option() -> None: @pytest.mark.parametrize( - "is_default_lock,invalid_reqs,invalid_constraints,uses_source_plugins,uses_project_ic", + "is_default_lock,invalid_reqs,invalid_interpreter_constraints,invalid_constraints_file,uses_source_plugins,uses_project_ic", [ - (is_default_lock, invalid_reqs, invalid_constraints, source_plugins, project_ics) + ( + is_default_lock, + invalid_reqs, + invalid_interpreter_constraints, + invalid_constraints_file, + source_plugins, + project_ics, + ) for is_default_lock in (True, False) for invalid_reqs in (True, False) - for invalid_constraints in (True, False) + for invalid_interpreter_constraints in (True, False) + for invalid_constraints_file in (True, False) for source_plugins in (True, False) for project_ics in (True, False) - if (invalid_reqs or invalid_constraints) + if (invalid_reqs or invalid_interpreter_constraints or invalid_constraints_file) ], ) def test_validate_tool_lockfiles( is_default_lock: bool, invalid_reqs: bool, - invalid_constraints: bool, + invalid_interpreter_constraints: bool, + invalid_constraints_file: bool, uses_source_plugins: bool, uses_project_ic: bool, caplog, ) -> None: runtime_interpreter_constraints = ( InterpreterConstraints(["==2.7.*"]) - if invalid_constraints + if invalid_interpreter_constraints else METADATA.valid_for_interpreter_constraints ) req_strings = ["bad-req"] if invalid_reqs else [str(r) for r in METADATA.requirements] @@ -113,6 +123,7 @@ def test_validate_tool_lockfiles( requirements, req_strings, create_python_setup(InvalidLockfileBehavior.warn), + constraints_file_path_and_hash=("c.txt", "xyz" if invalid_constraints_file else "abc"), ) def contains(msg: str, if_: bool) -> None: @@ -129,16 +140,18 @@ def contains(msg: str, if_: bool) -> None: ) contains(".source_plugins`, and", if_=invalid_reqs and uses_source_plugins) - contains("You have set interpreter constraints", if_=invalid_constraints) + contains("You have set interpreter constraints", if_=invalid_interpreter_constraints) contains( "determines its interpreter constraints based on your code's own constraints.", - if_=invalid_constraints and uses_project_ic, + if_=invalid_interpreter_constraints and uses_project_ic, ) contains( ".interpreter_constraints`, or by using a new custom lockfile.", - if_=invalid_constraints and not uses_project_ic, + if_=invalid_interpreter_constraints and not uses_project_ic, ) + contains("The constraints file at c.txt has changed", if_=invalid_constraints_file) + contains( "To generate a custom lockfile based on your current configuration", if_=is_default_lock ) @@ -148,22 +161,24 @@ def contains(msg: str, if_: bool) -> None: @pytest.mark.parametrize( - "invalid_reqs,invalid_constraints", + "invalid_reqs,invalid_interpreter_constraints,invalid_constraints_file", [ - (invalid_reqs, invalid_constraints) + (invalid_reqs, invalid_interpreter_constraints, invalid_constraints_file) for invalid_reqs in (True, False) - for invalid_constraints in (True, False) - if (invalid_reqs or invalid_constraints) + for invalid_interpreter_constraints in (True, False) + for invalid_constraints_file in (True, False) + if (invalid_reqs or invalid_interpreter_constraints or invalid_constraints_file) ], ) def test_validate_user_lockfiles( invalid_reqs: bool, - invalid_constraints: bool, + invalid_interpreter_constraints: bool, + invalid_constraints_file: bool, caplog, ) -> None: runtime_interpreter_constraints = ( InterpreterConstraints(["==2.7.*"]) - if invalid_constraints + if invalid_interpreter_constraints else METADATA.valid_for_interpreter_constraints ) req_strings = FrozenOrderedSet( @@ -186,6 +201,7 @@ def test_validate_user_lockfiles( lockfile, req_strings, create_python_setup(InvalidLockfileBehavior.warn), + constraints_file_path_and_hash=("c.txt", "xyz" if invalid_constraints_file else "abc"), ) def contains(msg: str, if_: bool = True) -> None: @@ -196,7 +212,11 @@ def contains(msg: str, if_: bool = True) -> None: "The targets depend on requirements that are not in the lockfile: ['bad-req']", if_=invalid_reqs, ) - contains("The targets use interpreter constraints", if_=invalid_constraints) + + contains("The targets use interpreter constraints", if_=invalid_interpreter_constraints) + + contains("The constraints file at c.txt has changed", if_=invalid_constraints_file) + contains("./pants generate-lockfiles --resolve=a`") diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 4e9bf5bf243..dd150be8038 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -556,7 +556,9 @@ def test_lockfile_validation(rule_runner: RuleRunner) -> None: # We create a lockfile that claims it works with no requirements. It should fail when we try # to build a PEX with a requirement. lock_content = PythonLockfileMetadata.new( - InterpreterConstraints(), set() + valid_for_interpreter_constraints=InterpreterConstraints(), + requirements=set(), + constraints_file_hash=None, ).add_header_to_lockfile(b"", regenerate_command="regen", delimeter="#") rule_runner.write_files({"lock.txt": lock_content.decode()}) diff --git a/src/python/pants/core/goals/update_build_files_test.py b/src/python/pants/core/goals/update_build_files_test.py index 7ab614159bc..d60fabd70c4 100644 --- a/src/python/pants/core/goals/update_build_files_test.py +++ b/src/python/pants/core/goals/update_build_files_test.py @@ -142,13 +142,18 @@ def test_goal_check_mode(generic_goal_rule_runner: RuleRunner) -> None: def test_find_python_interpreter_constraints_from_lockfile() -> None: + default_metadata = PythonLockfileMetadata.new( + valid_for_interpreter_constraints=InterpreterConstraints(["==2.7.*"]), + requirements=set(), + constraints_file_hash=None, + ) + def assert_ics( lockfile: str, expected: list[str], *, ics: RankedValue = RankedValue(Rank.HARDCODED, Black.default_interpreter_constraints), - metadata: PythonLockfileMetadata - | None = PythonLockfileMetadata.new(InterpreterConstraints(["==2.7.*"]), set()), + metadata: PythonLockfileMetadata | None = default_metadata, ) -> None: black = create_subsystem( Black, diff --git a/src/python/pants/core/util_rules/lockfile_metadata.py b/src/python/pants/core/util_rules/lockfile_metadata.py index 9f03a730c03..f5260fc5803 100644 --- a/src/python/pants/core/util_rules/lockfile_metadata.py +++ b/src/python/pants/core/util_rules/lockfile_metadata.py @@ -222,7 +222,8 @@ def metadata_version(self): `lockfile_metadata_version` """ for (scope, ver), cls in _concrete_metadata_classes.items(): - if isinstance(self, cls): + # Note that we do exact version matches so that authors can subclass earlier versions. + if type(self) is cls: return ver raise ValueError("Trying to serialize an unregistered `LockfileMetadata` subclass.") From d3ed2456b160f6ebde20714b7f794fcf5cf79621 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 8 Aug 2022 14:36:00 -0500 Subject: [PATCH 3/3] Fix test # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/subsystems/setup_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/subsystems/setup_test.py b/src/python/pants/backend/python/subsystems/setup_test.py index aecdcaaef34..b37abc33bf3 100644 --- a/src/python/pants/backend/python/subsystems/setup_test.py +++ b/src/python/pants/backend/python/subsystems/setup_test.py @@ -6,6 +6,7 @@ import pytest from pants.backend.python.subsystems.setup import PythonSetup +from pants.core.goals.generate_lockfiles import UnrecognizedResolveNamesError from pants.testutil.option_util import create_subsystem @@ -18,5 +19,5 @@ def create(resolves_to_ics: dict[str, list[str]]) -> dict[str, tuple[str, ...]]: ).resolves_to_interpreter_constraints assert create({"a": ["==3.7.*"]}) == {"a": ("==3.7.*",)} - with pytest.raises(KeyError): + with pytest.raises(UnrecognizedResolveNamesError): create({"fake": []})